sync(v1.6.0 / v1.0.9): synchronize full luminamoon monorepo, LSO-Eval v1.1.1 results, and LuminaV v1.6.0 bundle

.env.example

@@ -1,7 +1,7 @@
-# GitHub API Personal Access Tokens
-GITHUB_TOKEN=
-GITHUB_TOKEN_2=
-
-# Distribution & Registry Deployment Tokens
-HF_TOKEN=
-PYPI_TOKEN=
+# GitHub API Personal Access Tokens
+GITHUB_TOKEN=
+GITHUB_TOKEN_2=
+
+# Distribution & Registry Deployment Tokens
+HF_TOKEN=
+PYPI_TOKEN=

.github/dependabot.yml

@@ -1,37 +1,37 @@
-version: 2
-updates:
-  - package-ecosystem: "cargo"
-    directory: "/"
-    target-branch: "development"
-    schedule:
-      interval: "weekly"
-      day: "monday"
-    open-pull-requests-limit: 0
-    labels:
-      - "dependencies"
-      - "rust"
-      - "security"
-
-  - package-ecosystem: "pip"
-    directory: "/"
-    target-branch: "development"
-    schedule:
-      interval: "weekly"
-      day: "monday"
-    open-pull-requests-limit: 0
-    labels:
-      - "dependencies"
-      - "python"
-      - "security"
-
-  - package-ecosystem: "github-actions"
-    directory: "/"
-    target-branch: "development"
-    schedule:
-      interval: "weekly"
-      day: "monday"
-    open-pull-requests-limit: 0
-    labels:
-      - "dependencies"
-      - "github-actions"
-      - "security"
+version: 2
+updates:
+  - package-ecosystem: "cargo"
+    directory: "/"
+    target-branch: "development"
+    schedule:
+      interval: "weekly"
+      day: "monday"
+    open-pull-requests-limit: 0
+    labels:
+      - "dependencies"
+      - "rust"
+      - "security"
+
+  - package-ecosystem: "pip"
+    directory: "/"
+    target-branch: "development"
+    schedule:
+      interval: "weekly"
+      day: "monday"
+    open-pull-requests-limit: 0
+    labels:
+      - "dependencies"
+      - "python"
+      - "security"
+
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    target-branch: "development"
+    schedule:
+      interval: "weekly"
+      day: "monday"
+    open-pull-requests-limit: 0
+    labels:
+      - "dependencies"
+      - "github-actions"
+      - "security"

.github/workflows/ci.yml

@@ -1,309 +1,309 @@
-name: CI/CD Pipeline
-
-on:
-  push:
-    branches: [ "main", "master" ]
-    tags: [ "v*" ]
-  pull_request:
-    branches: [ "main", "master", "development" ]
-  release:
-    types: [published]
-  workflow_dispatch:
-
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-permissions:
-  contents: read
-
-jobs:
-  rust-quality:
-    name: Rust Quality & Parity (${{ matrix.os }})
-    if: github.event_name != 'release'
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        os: ${{ (startsWith(github.ref, 'refs/tags/v') || github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/main' || github.base_ref == 'main') && fromJSON('["ubuntu-latest", "windows-latest", "macos-latest"]') || fromJSON('["ubuntu-latest"]') }}
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v7
-
-      - name: Set up Python
-        uses: actions/setup-python@v7
-        with:
-          python-version: "3.12"
-
-      - name: Set up Rust toolchain
-        uses: dtolnay/rust-toolchain@stable
-        with:
-          components: clippy, rustfmt
-
-      - name: Rust Cache
-        uses: Swatinem/rust-cache@v2
-
-      - name: Verify Formatting (cargo fmt)
-        if: matrix.os == 'ubuntu-latest'
-        run: cargo fmt --check
-
-      - name: Cargo Clippy Lints
-        run: cargo clippy --workspace --all-targets -- -D warnings
-
-      - name: Cargo Unit & Parity Tests
-        run: cargo test --workspace --verbose
-
-  type-check:
-    name: Python Static Typing (Pyright)
-    if: github.event_name != 'release'
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v7
-
-      - name: Set up Python 3.14
-        uses: actions/setup-python@v7
-        with:
-          python-version: "3.14"
-
-      - name: Set up Node.js
-        uses: actions/setup-node@v7
-        with:
-          node-version: "24"
-
-      - name: Install Pyright Dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install "numpy>=2.0.0"
-
-      - name: Run Pyright
-        run: npx pyright python/luminamoon
-
-      - name: Verify Universal Fallback Wheel Builder
-        run: |
-          python tools/build_universal_wheel.py --out dist
-          python -c "import zipfile, sys, tempfile; td = tempfile.TemporaryDirectory(); import glob; whl = glob.glob('dist/luminamoon-*-py3-none-any.whl')[0]; z = zipfile.ZipFile(whl); z.extractall(td.name); sys.path.insert(0, td.name); import luminamoon.optimizer.luminav as lum; import numpy as np; p = np.zeros(5, dtype=np.float32); opt = lum.LuminaV([p], lr=1e-3); opt.step(grads=[np.ones(5, dtype=np.float32)]); assert opt.state[p]['step'] == 1; print('Universal fallback verified successfully')"
-
-  python-matrix:
-    name: Python Tests (${{ matrix.os }}, Py${{ matrix.python-version }})
-    if: github.event_name != 'release'
-    needs: [rust-quality]
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        os: ${{ (startsWith(github.ref, 'refs/tags/v') || github.event_name == 'workflow_dispatch') && fromJSON('["ubuntu-latest", "windows-latest", "macos-latest"]') || ((github.ref == 'refs/heads/main' || github.base_ref == 'main') && fromJSON('["ubuntu-latest", "windows-latest"]') || fromJSON('["ubuntu-latest"]')) }}
-        python-version: ${{ (startsWith(github.ref, 'refs/tags/v') || github.event_name == 'workflow_dispatch') && fromJSON('["3.10", "3.11", "3.12", "3.13", "3.14"]') || fromJSON('["3.14"]') }}
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v7
-
-      - name: Set up Rust toolchain
-        uses: dtolnay/rust-toolchain@stable
-
-      - name: Rust Cache
-        uses: Swatinem/rust-cache@v2
-
-      - name: Set up Python
-        uses: actions/setup-python@v7
-        with:
-          python-version: ${{ matrix.python-version }}
-          allow-prereleases: true
-          cache: "pip"
-
-      - name: Upgrade Pip & Install Build Tools
-        run: |
-          python -m pip install --upgrade pip
-          pip install maturin "numpy>=2.0.0" pytest
-
-      - name: Build & Install LuminaMoon (Native CPython Extension)
-        run: python tools/build_wheel.py --install
-
-      - name: Run Standalone NumPy Verification
-        run: python optimizer/tests/test_standalone_numpy.py
-
-      - name: Run Data Modular Verification
-        run: python -B tests/test_data_modular.py
-
-      - name: Run Adversarial String Verification
-        run: python -B tests/test_gpu_vs_cpu_strings.py
-
-      - name: Install Optional PyTorch Dependency
-        id: install-torch
-        run: python -c "import os, subprocess, sys; r = subprocess.run([sys.executable, '-m', 'pip', 'install', 'torch>=2.0.0']); open(os.environ['GITHUB_OUTPUT'], 'a', encoding='utf-8').write(f'installed={str(r.returncode == 0).lower()}\n'); print('PyTorch installed:', r.returncode == 0)"
-
-
-      - name: Run PyTorch End-to-End Training Tests
-        if: steps.install-torch.outputs.installed == 'true'
-        run: python optimizer/tests/test_end_to_end_training.py
-
-      - name: Run LSO-Eval Vision (V-Eval) Protocol Verification
-        if: steps.install-torch.outputs.installed == 'true'
-        run: python eval/v_eval_suite.py --verify-architecture
-
-      - name: Verify Universal Dynamic Fallback
-        run: |
-          python -c "import luminamoon.optimizer.luminav as lum; lum._luminamoon_rs = None; import numpy as np; p = np.zeros(10, dtype=np.float32); opt = lum.LuminaV([p], lr=1e-3); opt.step(grads=[np.ones(10, dtype=np.float32)]); assert opt.state[p]['step'] == 1; print('Dynamic fallback verified successfully')"
-
-  free-threaded:
-    name: Free-Threaded No-GIL (${{ matrix.python-version }})
-    if: github.event_name != 'release'
-    needs: [rust-quality]
-    runs-on: ubuntu-latest
-    continue-on-error: true
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version: ["3.14t", "3.15t"]
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v7
-
-      - name: Set up Rust toolchain
-        uses: dtolnay/rust-toolchain@stable
-
-      - name: Rust Cache
-        uses: Swatinem/rust-cache@v2
-
-      - name: Set up Python (Free-Threaded)
-        uses: actions/setup-python@v7
-        with:
-          python-version: ${{ matrix.python-version }}
-          allow-prereleases: true
-
-      - name: Install Maturin
-        run: |
-          python -m pip install --upgrade pip
-          pip install maturin
-
-      - name: Verify abi3t Wheel Compilation & Import
-        run: |
-          maturin build --release -i python --out dist
-          pip install dist/*.whl
-          python -c "import luminamoon.optimizer._luminamoon_optimizer_rs, sys; assert not sys._is_gil_enabled(), 'GIL must remain disabled'; print('Free-threaded abi3t extension verified successfully')"
-
-  release-build:
-    name: Distribution Wheel Build (${{ matrix.os }})
-    needs: [python-matrix, type-check]
-    if: (startsWith(github.ref, 'refs/tags/v') || github.event_name == 'workflow_dispatch') && github.event_name != 'release'
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v7
-
-      - name: Set up Python for Build Scripts
-        if: matrix.os == 'ubuntu-latest' || startsWith(github.ref, 'refs/tags/v')
-        uses: actions/setup-python@v7
-        with:
-          python-version: "3.14"
-
-      - name: Sync version from Git Tag
-        if: startsWith(github.ref, 'refs/tags/v')
-        run: python tools/sync_version.py "${{ github.ref_name }}"
-
-      - name: Build Universal Pure-Python Fallback Wheel (py3-none-any)
-        if: matrix.os == 'ubuntu-latest'
-        run: python tools/build_universal_wheel.py --out dist
-
-      - name: Set up Rust toolchain
-        uses: dtolnay/rust-toolchain@stable
-
-      - name: Build sdist (Source Distribution)
-        if: matrix.os == 'ubuntu-latest'
-        uses: PyO3/maturin-action@v1
-        with:
-          command: sdist
-          args: --out dist
-
-      - name: Build Wheel via Maturin
-        uses: PyO3/maturin-action@v1
-        with:
-          command: build
-          args: --release --out dist
-          manylinux: auto
-
-      - name: Upload Artifacts
-        uses: actions/upload-artifact@v7
-        with:
-          name: wheels-${{ matrix.os }}
-          path: dist/*
-
-  draft-release:
-    name: Create Draft GitHub Release
-    needs: [release-build]
-    if: startsWith(github.ref, 'refs/tags/v') && github.event_name != 'release'
-    runs-on: ubuntu-latest
-    permissions:
-      contents: write
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v7
-
-      - name: Download all built wheels
-        uses: actions/download-artifact@v8
-        with:
-          pattern: wheels-*
-          merge-multiple: true
-          path: dist
-
-      - name: Extract Structured Release Notes from Software Changelog
-        env:
-          RELEASE_TAG: ${{ github.ref_name }}
-        run: |
-          python3 -c '
-          import os, pathlib, re
-          tag = os.environ["RELEASE_TAG"].lstrip("v")
-          text = pathlib.Path("changelogs/SOFTWARE_CHANGELOG.md").read_text(encoding="utf-8")
-          pattern = rf"^## \[{re.escape(tag)}\].*?(?=^## \[|\Z)"
-          match = re.search(pattern, text, flags=re.MULTILINE | re.DOTALL)
-          body = re.sub(r"\n+---\s*$", "", match.group(0).strip()) if match else f"## [{tag}]\n\nRelease of Loomy (`luminamoon`) v{tag}."
-          header = (
-              f"{body}\n\n---\n\n"
-              "## Installation\n\n"
-              f"```bash\npip install --upgrade luminamoon=={tag}\n```\n"
-          )
-          pathlib.Path("RELEASE_NOTES.md").write_text(header, encoding="utf-8")
-          '
-
-      - name: Create Draft GitHub Release
-        uses: softprops/action-gh-release@v2
-        with:
-          name: "${{ github.ref_name }}"
-          body_path: RELEASE_NOTES.md
-          files: dist/*
-          draft: true
-          prerelease: ${{ contains(github.ref_name, 'a') || contains(github.ref_name, 'b') || contains(github.ref_name, 'rc') }}
-          make_latest: ${{ !contains(github.ref_name, 'a') && !contains(github.ref_name, 'b') && !contains(github.ref_name, 'rc') }}
-          generate_release_notes: true
-
-  publish-pypi:
-    name: Publish Wheels to PyPI
-    needs: [draft-release]
-    if: always() && (github.event_name == 'release' && github.event.action == 'published')
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      id-token: write
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v7
-
-      - name: Download release assets via GitHub CLI
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          RELEASE_TAG: ${{ github.event.release.tag_name || github.ref_name }}
-        run: |
-          mkdir -p dist
-          gh release download "$RELEASE_TAG" --dir dist --pattern "*.whl" --pattern "*.tar.gz"
-          ls -la dist
-
-      - name: Publish Distribution to PyPI
-        uses: pypa/gh-action-pypi-publish@release/v1
-        with:
-          password: ${{ secrets.PYPI_API_TOKEN }}
-          packages-dir: dist/
+name: CI/CD Pipeline
+
+on:
+  push:
+    branches: [ "main", "master" ]
+    tags: [ "v*" ]
+  pull_request:
+    branches: [ "main", "master", "development" ]
+  release:
+    types: [published]
+  workflow_dispatch:
+
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  rust-quality:
+    name: Rust Quality & Parity (${{ matrix.os }})
+    if: github.event_name != 'release'
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ${{ (startsWith(github.ref, 'refs/tags/v') || github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/main' || github.base_ref == 'main') && fromJSON('["ubuntu-latest", "windows-latest", "macos-latest"]') || fromJSON('["ubuntu-latest"]') }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v7
+
+      - name: Set up Python
+        uses: actions/setup-python@v7
+        with:
+          python-version: "3.12"
+
+      - name: Set up Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          components: clippy, rustfmt
+
+      - name: Rust Cache
+        uses: Swatinem/rust-cache@v2
+
+      - name: Verify Formatting (cargo fmt)
+        if: matrix.os == 'ubuntu-latest'
+        run: cargo fmt --check
+
+      - name: Cargo Clippy Lints
+        run: cargo clippy --workspace --all-targets -- -D warnings
+
+      - name: Cargo Unit & Parity Tests
+        run: cargo test --workspace --verbose
+
+  type-check:
+    name: Python Static Typing (Pyright)
+    if: github.event_name != 'release'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v7
+
+      - name: Set up Python 3.14
+        uses: actions/setup-python@v7
+        with:
+          python-version: "3.14"
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v7
+        with:
+          node-version: "24"
+
+      - name: Install Pyright Dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install "numpy>=2.0.0"
+
+      - name: Run Pyright
+        run: npx pyright python/luminamoon
+
+      - name: Verify Universal Fallback Wheel Builder
+        run: |
+          python tools/build_universal_wheel.py --out dist
+          python -c "import zipfile, sys, tempfile; td = tempfile.TemporaryDirectory(); import glob; whl = glob.glob('dist/luminamoon-*-py3-none-any.whl')[0]; z = zipfile.ZipFile(whl); z.extractall(td.name); sys.path.insert(0, td.name); import luminamoon.optimizer.luminav as lum; import numpy as np; p = np.zeros(5, dtype=np.float32); opt = lum.LuminaV([p], lr=1e-3); opt.step(grads=[np.ones(5, dtype=np.float32)]); assert opt.state[p]['step'] == 1; print('Universal fallback verified successfully')"
+
+  python-matrix:
+    name: Python Tests (${{ matrix.os }}, Py${{ matrix.python-version }})
+    if: github.event_name != 'release'
+    needs: [rust-quality]
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ${{ (startsWith(github.ref, 'refs/tags/v') || github.event_name == 'workflow_dispatch') && fromJSON('["ubuntu-latest", "windows-latest", "macos-latest"]') || ((github.ref == 'refs/heads/main' || github.base_ref == 'main') && fromJSON('["ubuntu-latest", "windows-latest"]') || fromJSON('["ubuntu-latest"]')) }}
+        python-version: ${{ (startsWith(github.ref, 'refs/tags/v') || github.event_name == 'workflow_dispatch') && fromJSON('["3.10", "3.11", "3.12", "3.13", "3.14"]') || fromJSON('["3.14"]') }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v7
+
+      - name: Set up Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+
+      - name: Rust Cache
+        uses: Swatinem/rust-cache@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v7
+        with:
+          python-version: ${{ matrix.python-version }}
+          allow-prereleases: true
+          cache: "pip"
+
+      - name: Upgrade Pip & Install Build Tools
+        run: |
+          python -m pip install --upgrade pip
+          pip install maturin "numpy>=2.0.0" pytest
+
+      - name: Build & Install LuminaMoon (Native CPython Extension)
+        run: python tools/build_wheel.py --install
+
+      - name: Run Standalone NumPy Verification
+        run: python optimizer/tests/test_standalone_numpy.py
+
+      - name: Run Data Modular Verification
+        run: python -B tests/test_data_modular.py
+
+      - name: Run Adversarial String Verification
+        run: python -B tests/test_gpu_vs_cpu_strings.py
+
+      - name: Install Optional PyTorch Dependency
+        id: install-torch
+        run: python -c "import os, subprocess, sys; r = subprocess.run([sys.executable, '-m', 'pip', 'install', 'torch>=2.0.0']); open(os.environ['GITHUB_OUTPUT'], 'a', encoding='utf-8').write(f'installed={str(r.returncode == 0).lower()}\n'); print('PyTorch installed:', r.returncode == 0)"
+
+
+      - name: Run PyTorch End-to-End Training Tests
+        if: steps.install-torch.outputs.installed == 'true'
+        run: python optimizer/tests/test_end_to_end_training.py
+
+      - name: Run LSO-Eval Vision (V-Eval) Protocol Verification
+        if: steps.install-torch.outputs.installed == 'true'
+        run: python eval/v_eval_suite.py --verify-architecture
+
+      - name: Verify Universal Dynamic Fallback
+        run: |
+          python -c "import luminamoon.optimizer.luminav as lum; lum._luminamoon_rs = None; import numpy as np; p = np.zeros(10, dtype=np.float32); opt = lum.LuminaV([p], lr=1e-3); opt.step(grads=[np.ones(10, dtype=np.float32)]); assert opt.state[p]['step'] == 1; print('Dynamic fallback verified successfully')"
+
+  free-threaded:
+    name: Free-Threaded No-GIL (${{ matrix.python-version }})
+    if: github.event_name != 'release'
+    needs: [rust-quality]
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.14t", "3.15t"]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v7
+
+      - name: Set up Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+
+      - name: Rust Cache
+        uses: Swatinem/rust-cache@v2
+
+      - name: Set up Python (Free-Threaded)
+        uses: actions/setup-python@v7
+        with:
+          python-version: ${{ matrix.python-version }}
+          allow-prereleases: true
+
+      - name: Install Maturin
+        run: |
+          python -m pip install --upgrade pip
+          pip install maturin
+
+      - name: Verify abi3t Wheel Compilation & Import
+        run: |
+          maturin build --release -i python --out dist
+          pip install dist/*.whl
+          python -c "import luminamoon.optimizer._luminamoon_optimizer_rs, sys; assert not sys._is_gil_enabled(), 'GIL must remain disabled'; print('Free-threaded abi3t extension verified successfully')"
+
+  release-build:
+    name: Distribution Wheel Build (${{ matrix.os }})
+    needs: [python-matrix, type-check]
+    if: (startsWith(github.ref, 'refs/tags/v') || github.event_name == 'workflow_dispatch') && github.event_name != 'release'
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v7
+
+      - name: Set up Python for Build Scripts
+        if: matrix.os == 'ubuntu-latest' || startsWith(github.ref, 'refs/tags/v')
+        uses: actions/setup-python@v7
+        with:
+          python-version: "3.14"
+
+      - name: Sync version from Git Tag
+        if: startsWith(github.ref, 'refs/tags/v')
+        run: python tools/sync_version.py "${{ github.ref_name }}"
+
+      - name: Build Universal Pure-Python Fallback Wheel (py3-none-any)
+        if: matrix.os == 'ubuntu-latest'
+        run: python tools/build_universal_wheel.py --out dist
+
+      - name: Set up Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+
+      - name: Build sdist (Source Distribution)
+        if: matrix.os == 'ubuntu-latest'
+        uses: PyO3/maturin-action@v1
+        with:
+          command: sdist
+          args: --out dist
+
+      - name: Build Wheel via Maturin
+        uses: PyO3/maturin-action@v1
+        with:
+          command: build
+          args: --release --out dist
+          manylinux: auto
+
+      - name: Upload Artifacts
+        uses: actions/upload-artifact@v7
+        with:
+          name: wheels-${{ matrix.os }}
+          path: dist/*
+
+  draft-release:
+    name: Create Draft GitHub Release
+    needs: [release-build]
+    if: startsWith(github.ref, 'refs/tags/v') && github.event_name != 'release'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v7
+
+      - name: Download all built wheels
+        uses: actions/download-artifact@v8
+        with:
+          pattern: wheels-*
+          merge-multiple: true
+          path: dist
+
+      - name: Extract Structured Release Notes from Software Changelog
+        env:
+          RELEASE_TAG: ${{ github.ref_name }}
+        run: |
+          python3 -c '
+          import os, pathlib, re
+          tag = os.environ["RELEASE_TAG"].lstrip("v")
+          text = pathlib.Path("changelogs/SOFTWARE_CHANGELOG.md").read_text(encoding="utf-8")
+          pattern = rf"^## \[{re.escape(tag)}\].*?(?=^## \[|\Z)"
+          match = re.search(pattern, text, flags=re.MULTILINE | re.DOTALL)
+          body = re.sub(r"\n+---\s*$", "", match.group(0).strip()) if match else f"## [{tag}]\n\nRelease of Loomy (`luminamoon`) v{tag}."
+          header = (
+              f"{body}\n\n---\n\n"
+              "## Installation\n\n"
+              f"```bash\npip install --upgrade luminamoon=={tag}\n```\n"
+          )
+          pathlib.Path("RELEASE_NOTES.md").write_text(header, encoding="utf-8")
+          '
+
+      - name: Create Draft GitHub Release
+        uses: softprops/action-gh-release@v2
+        with:
+          name: "${{ github.ref_name }}"
+          body_path: RELEASE_NOTES.md
+          files: dist/*
+          draft: true
+          prerelease: ${{ contains(github.ref_name, 'a') || contains(github.ref_name, 'b') || contains(github.ref_name, 'rc') }}
+          make_latest: ${{ !contains(github.ref_name, 'a') && !contains(github.ref_name, 'b') && !contains(github.ref_name, 'rc') }}
+          generate_release_notes: true
+
+  publish-pypi:
+    name: Publish Wheels to PyPI
+    needs: [draft-release]
+    if: always() && (github.event_name == 'release' && github.event.action == 'published')
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v7
+
+      - name: Download release assets via GitHub CLI
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          RELEASE_TAG: ${{ github.event.release.tag_name || github.ref_name }}
+        run: |
+          mkdir -p dist
+          gh release download "$RELEASE_TAG" --dir dist --pattern "*.whl" --pattern "*.tar.gz"
+          ls -la dist
+
+      - name: Publish Distribution to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.PYPI_API_TOKEN }}
+          packages-dir: dist/
           skip-existing: true

.github/workflows/docs.yml

@@ -1,83 +1,83 @@
-name: Documentation Portal (GitHub Pages)
-
-on:
-  push:
-    branches: [ "main" ]
-  pull_request:
-    branches: [ "main", "development" ]
-    paths:
-      - "docs/**"
-      - "mkdocs.yml"
-      - "README.md"
-      - "**/README.md"
-      - "changelogs/**"
-      - "SECURITY.md"
-      - "tools/prepare_mkdocs.py"
-      - "tools/prune_deployments.py"
-      - ".github/workflows/docs.yml"
-  workflow_dispatch:
-
-permissions:
-  contents: read
-  pages: write
-  id-token: write
-  deployments: write
-
-concurrency:
-  group: "pages-${{ github.ref }}"
-  cancel-in-progress: true
-
-jobs:
-  build:
-    name: Build MkDocs Material Site
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v7
-
-      - name: Set up Python
-        uses: actions/setup-python@v7
-        with:
-          python-version: "3.14"
-          cache: "pip"
-
-      - name: Install MkDocs Material
-        run: |
-          python -m pip install --upgrade pip
-          pip install "mkdocs-material>=9.5.0"
-
-      - name: Prepare Documentation Stage & Normalize Math/Links
-        run: python tools/prepare_mkdocs.py
-
-      - name: Build Static Documentation Site
-        run: mkdocs build
-
-      - name: Upload GitHub Pages Artifact
-        if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main'
-        uses: actions/upload-pages-artifact@v5
-        with:
-          path: site
-
-  deploy:
-    name: Deploy to GitHub Pages
-    if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main'
-    needs: build
-    runs-on: ubuntu-latest
-    environment:
-      name: github-pages
-      url: ${{ steps.deployment.outputs.page_url }}
-    steps:
-      - name: Deploy to GitHub Pages
-        id: deployment
-        uses: actions/deploy-pages@v5
-
-      - name: Checkout repository for tools
-        if: success()
-        uses: actions/checkout@v7
-
-      - name: Prune Older Deployments (Retention Max 3)
-        if: success()
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          GH_REPO: ${{ github.repository }}
-        run: python tools/prune_deployments.py --retention 3
+name: Documentation Portal (GitHub Pages)
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main", "development" ]
+    paths:
+      - "docs/**"
+      - "mkdocs.yml"
+      - "README.md"
+      - "**/README.md"
+      - "changelogs/**"
+      - "SECURITY.md"
+      - "tools/prepare_mkdocs.py"
+      - "tools/prune_deployments.py"
+      - ".github/workflows/docs.yml"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+  deployments: write
+
+concurrency:
+  group: "pages-${{ github.ref }}"
+  cancel-in-progress: true
+
+jobs:
+  build:
+    name: Build MkDocs Material Site
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v7
+
+      - name: Set up Python
+        uses: actions/setup-python@v7
+        with:
+          python-version: "3.14"
+          cache: "pip"
+
+      - name: Install MkDocs Material
+        run: |
+          python -m pip install --upgrade pip
+          pip install "mkdocs-material>=9.5.0"
+
+      - name: Prepare Documentation Stage & Normalize Math/Links
+        run: python tools/prepare_mkdocs.py
+
+      - name: Build Static Documentation Site
+        run: mkdocs build
+
+      - name: Upload GitHub Pages Artifact
+        if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main'
+        uses: actions/upload-pages-artifact@v5
+        with:
+          path: site
+
+  deploy:
+    name: Deploy to GitHub Pages
+    if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main'
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v5
+
+      - name: Checkout repository for tools
+        if: success()
+        uses: actions/checkout@v7
+
+      - name: Prune Older Deployments (Retention Max 3)
+        if: success()
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GH_REPO: ${{ github.repository }}
+        run: python tools/prune_deployments.py --retention 3

.github/workflows/hf_sync.yml

@@ -1,49 +1,49 @@
-name: Hugging Face Hub Auto-Sync
-
-on:
-  push:
-    branches: [ "main" ]
-    tags: [ "v*" ]
-  pull_request:
-    branches: [ "main" ]
-  release:
-    types: [published]
-  workflow_dispatch:
-
-concurrency:
-  group: "hf-sync-${{ github.ref }}"
-  cancel-in-progress: true
-
-permissions:
-  contents: read
-
-jobs:
-  sync-huggingface:
-    name: Stage & Sync Hugging Face Repositories
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v7
-
-      - name: Set up Python
-        uses: actions/setup-python@v7
-        with:
-          python-version: "3.14"
-          cache: "pip"
-
-      - name: Install Dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install "huggingface_hub>=0.25.0" --extra-index-url https://download.pytorch.org/whl/cpu torch
-
-      - name: Stage & Verify Hugging Face Bundles (PR Check)
-        if: github.event_name == 'pull_request'
-        run: python tools/sync_huggingface.py
-
-      - name: Stage, Verify & Push to Hugging Face Hub (main / release)
-        if: github.event_name != 'pull_request' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v') || github.event_name == 'release' || github.event_name == 'workflow_dispatch')
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN }}
-          HF_ORG: ${{ secrets.HF_ORG }}
-          HF_USERNAME: ${{ secrets.HF_USERNAME }}
-        run: python tools/sync_huggingface.py --push
+name: Hugging Face Hub Auto-Sync
+
+on:
+  push:
+    branches: [ "main" ]
+    tags: [ "v*" ]
+  pull_request:
+    branches: [ "main" ]
+  release:
+    types: [published]
+  workflow_dispatch:
+
+concurrency:
+  group: "hf-sync-${{ github.ref }}"
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  sync-huggingface:
+    name: Stage & Sync Hugging Face Repositories
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v7
+
+      - name: Set up Python
+        uses: actions/setup-python@v7
+        with:
+          python-version: "3.14"
+          cache: "pip"
+
+      - name: Install Dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install "huggingface_hub>=0.25.0" --extra-index-url https://download.pytorch.org/whl/cpu torch
+
+      - name: Stage & Verify Hugging Face Bundles (PR Check)
+        if: github.event_name == 'pull_request'
+        run: python tools/sync_huggingface.py
+
+      - name: Stage, Verify & Push to Hugging Face Hub (main / release)
+        if: github.event_name != 'pull_request' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v') || github.event_name == 'release' || github.event_name == 'workflow_dispatch')
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+          HF_ORG: ${{ secrets.HF_ORG }}
+          HF_USERNAME: ${{ secrets.HF_USERNAME }}
+        run: python tools/sync_huggingface.py --push

.github/workflows/security.yml

@@ -1,91 +1,91 @@
-name: Security & Vulnerability Audit (CVE / CWE / SAST)
-
-on:
-  push:
-    branches: [ "main" ]
-  pull_request:
-    branches: [ "main", "development" ]
-  schedule:
-    - cron: "0 3 * * 1"
-  workflow_dispatch:
-
-permissions:
-  contents: read
-
-concurrency:
-  group: "security-${{ github.ref }}"
-  cancel-in-progress: true
-
-jobs:
-  codeql-cwe-sast:
-    name: CodeQL SAST & CWE Analysis (${{ matrix.language }})
-    runs-on: ubuntu-latest
-    permissions:
-      actions: read
-      contents: read
-      security-events: write
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - language: python
-            build-mode: none
-          - language: actions
-            build-mode: none
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v7
-
-      - name: Initialize CodeQL (CWE Static Analyzer)
-        uses: github/codeql-action/init@v4
-        with:
-          languages: ${{ matrix.language }}
-          build-mode: ${{ matrix.build-mode }}
-
-      - name: Perform CodeQL CWE Analysis & Upload SARIF
-        uses: github/codeql-action/analyze@v4
-        with:
-          category: "/language:${{ matrix.language }}"
-
-  supply-chain-cve-audit:
-    name: RustSec, OSV & PyPA CVE Supply-Chain Audit
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v7
-
-      - name: Set up Python
-        uses: actions/setup-python@v7
-        with:
-          python-version: "3.14"
-          cache: "pip"
-
-      - name: Install PyPA pip-audit & Build Tools
-        run: |
-          python -m pip install --upgrade pip
-          pip install "pip-audit>=2.7.0"
-
-      - name: Audit Cargo.lock (RustSec/OSV/CVE) & Repository Secret Hygiene
-        run: python tools/security_audit.py
-
-      - name: Audit Python Environment via PyPA pip-audit (OSV / PYSEC / CVE)
-        run: |
-          python tools/build_universal_wheel.py --out dist
-          pip install dist/luminamoon-*-py3-none-any.whl
-          pip-audit --desc on
-
-  dependency-review:
-    name: PR Dependency & License Vulnerability Gate
-    if: github.event_name == 'pull_request'
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      pull-requests: read
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v7
-
-      - name: Dependency Review (CVE / GHSA Severity Gate)
-        uses: actions/dependency-review-action@v5
-        with:
-          fail-on-severity: moderate
+name: Security & Vulnerability Audit (CVE / CWE / SAST)
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main", "development" ]
+  schedule:
+    - cron: "0 3 * * 1"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: "security-${{ github.ref }}"
+  cancel-in-progress: true
+
+jobs:
+  codeql-cwe-sast:
+    name: CodeQL SAST & CWE Analysis (${{ matrix.language }})
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - language: python
+            build-mode: none
+          - language: actions
+            build-mode: none
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v7
+
+      - name: Initialize CodeQL (CWE Static Analyzer)
+        uses: github/codeql-action/init@v4
+        with:
+          languages: ${{ matrix.language }}
+          build-mode: ${{ matrix.build-mode }}
+
+      - name: Perform CodeQL CWE Analysis & Upload SARIF
+        uses: github/codeql-action/analyze@v4
+        with:
+          category: "/language:${{ matrix.language }}"
+
+  supply-chain-cve-audit:
+    name: RustSec, OSV & PyPA CVE Supply-Chain Audit
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v7
+
+      - name: Set up Python
+        uses: actions/setup-python@v7
+        with:
+          python-version: "3.14"
+          cache: "pip"
+
+      - name: Install PyPA pip-audit & Build Tools
+        run: |
+          python -m pip install --upgrade pip
+          pip install "pip-audit>=2.7.0"
+
+      - name: Audit Cargo.lock (RustSec/OSV/CVE) & Repository Secret Hygiene
+        run: python tools/security_audit.py
+
+      - name: Audit Python Environment via PyPA pip-audit (OSV / PYSEC / CVE)
+        run: |
+          python tools/build_universal_wheel.py --out dist
+          pip install dist/luminamoon-*-py3-none-any.whl
+          pip-audit --desc on
+
+  dependency-review:
+    name: PR Dependency & License Vulnerability Gate
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pull-requests: read
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v7
+
+      - name: Dependency Review (CVE / GHSA Severity Gate)
+        uses: actions/dependency-review-action@v5
+        with:
+          fail-on-severity: moderate

.gitignore

@@ -1,54 +1,54 @@
-/target/
-/optimizer/target/
-**/target/
-__pycache__/
-*.py[cod]
-*$py.class
-*.so
-*.pyd
-*.dll
-*.dylib
-*.exe
-*.lib
-*.a
-*.o
-*.obj
-*.pdb
-*.exp
-.pytest_cache/
-.ruff_cache/
-.mypy_cache/
-build/
-dist/
-wheels/
-*.whl
-*.egg-info/
-.venv/
-venv/
-.env
-.env.*
-!.env.example
-.DS_Store
-Thumbs.db
-*.log
-.coverage
-htmlcov/
-.benchmarks/
-.vscode/
-.idea/
-optimizer/tools/aot_compiler/glslang/
-.luminamoon_cache/
-*.meta
-*.meta.dedup
-!eval/artifacts/**/*.meta
-!eval/artifacts/**/*.meta.dedup
-x.py
-logs/
-site/
-docs/overview.md
-docs/security.md
-docs/lso_eval.md
-docs/optimize/standalone_guide.md
-docs/dataloader/standalone_guide.md
-docs/changelogs/
-scratch/
+/target/
+/optimizer/target/
+**/target/
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+*.pyd
+*.dll
+*.dylib
+*.exe
+*.lib
+*.a
+*.o
+*.obj
+*.pdb
+*.exp
+.pytest_cache/
+.ruff_cache/
+.mypy_cache/
+build/
+dist/
+wheels/
+*.whl
+*.egg-info/
+.venv/
+venv/
+.env
+.env.*
+!.env.example
+.DS_Store
+Thumbs.db
+*.log
+.coverage
+htmlcov/
+.benchmarks/
+.vscode/
+.idea/
+optimizer/tools/aot_compiler/glslang/
+.luminamoon_cache/
+*.meta
+*.meta.dedup
+!eval/artifacts/**/*.meta
+!eval/artifacts/**/*.meta.dedup
+x.py
+logs/
+site/
+docs/overview.md
+docs/security.md
+docs/lso_eval.md
+docs/optimize/standalone_guide.md
+docs/dataloader/standalone_guide.md
+docs/changelogs/
+scratch/

CHANGELOG.md

@@ -1,430 +1,430 @@
-# LuminaV Optimizer Algorithm Changelog
-
-All notable changes to the LuminaV Optimizer algorithm are documented in this file.
-
-The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
-
-* [Software Package Changelog (`luminamoon`)](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/changelogs/SOFTWARE_CHANGELOG.md)
-
----
-
-## [1.6.0] - 2026-11-03
-
-### Added
-- **T-Family (Triangle Family) tanh-free Modes (`buffer="T1"` & `buffer="T2"`)**:
-  * **`buffer="T2"` (2-Buffer Triangle Family Mode)**: Keeps the 2-buffer state (`exp_avg` and `exp_avg_sq`) without using $\tanh$, computing the gradient-momentum residual $d_t = g_t - m _ {t-1}$ and bounding the lookahead update $\tilde{m}_t$ via the **Triangle Inequality Envelope** $\mathcal{E}_t^{(\text{T2})}$:
-
-    $\displaystyle d_t = g_t - m _ {t-1}, \qquad v_t = \beta_2 v _ {t-1} + (1 - \beta_2) d_t^2$
-
-    $\displaystyle \tilde{m} _ t = w_1 m _ {t-1} + w_2 g_t, \qquad w_1 = \beta_1^2, \qquad w_2 = 1 - \beta_1^2$
-
-    $\displaystyle \mathcal{E} _ t^{(\text{T2})} = w_1 |m _ {t-1}| + w_2 |g_t| + 0.5\lambda |d_t| + 0.5\lambda \sigma_t + \epsilon _ {\text{eff}}, \qquad u_t^{(\text{T2})} = \frac{\tilde{m} _ t}{\mathcal{E} _ t^{(\text{T2})}} \in (-1, 1)$
-
-    By the Triangle Inequality ($|w_1 m _ {t-1} + w_2 g_t| \le w_1 |m _ {t-1}| + w_2 |g_t| \le \mathcal{E} _ t^{(\text{T2})}$), $|u_t^{(\text{T2})}| < 1$ is guaranteed without evaluating $\tanh$. The term $0.5\lambda |d_t|$ acts as a **triangle-defect penalty**, and $0.5\lambda \sigma_t$ provides belief-variance damping.
-  * **`buffer="T1"` (1-Buffer Triangle Family Mode)**: Combines the single-buffer **Triangle-Defect** denominator $\mathcal{E} _ t^{(\text{T1})} = w_1 |m _ {t-1}| + w_2 |g_t| + \lambda |d_t| + \epsilon _ {\text{eff}}$ (using residual $d_t = g_t - m _ {t-1}$) with the rational update bounds below, reducing optimizer state memory by 50% via the Triangle Inequality.
-  * **Rational Update Bounds (for `T1` & `T2`)**: Replaces $\tanh$ and $\sqrt{\cdot}$ in the coordinate, radial, and FP16 update bounds with simple rational formulas:
-
-    $\displaystyle b_i = \mathrm{bound} _ {\text{ratio}} \cdot (|w_i| + 10^{-2}), \qquad \Delta w_i = \frac{b_i (b_i + |s_i|) s_i}{b_i (b_i + |s_i|) + s_i^2}, \qquad \mathrm{scale} _ {\text{radial}} = \frac{1 + r}{1 + r + r^2}$
-  * Keeps 100% backward compatibility and identical outputs for existing modes (`buffer=1`, `buffer=2`, and `version="1.5.5"`).
-
----
-
-## [1.5.5] - 2026-10-29
-
-### Added
-- **Contiguous Bucket Fusion (`fuse_tensors: bool = False`)**:
-  * Added opt-in contiguous parameter and state coalescing (`_build_contiguous_buckets`) that groups parameters within each `(param_group, device, dtype)` domain $\mathcal{B}_{g,d,\tau}$ into contiguous 1D storage (`flat_p`, `flat_g`, `flat_m`, `flat_v`, `flat_mp`) and rebinds `p.data`, `p.grad`, and `self.state[p]` entries as zero-copy views (`flat_*[offset : offset + n].view(p.shape)`).
-  * Eliminates per-tensor CPU-to-GPU kernel launch latency on fragmented vision and edge architectures by executing an entire bucket in a single fused kernel dispatch while computing `cautious_scale` and `radial` trust bounds across the bucket domain $\mathcal{B}_{g,d,\tau}$.
-  * Added in-place `zero_grad()` view preservation (`flat_g.zero_()`) and full `state_dict()` / `load_state_dict()` checkpoint compatibility without leaking temporary buffer keys (LSO-Eval Stage II compliant).
-- **Triton JIT Pre-Warmup (`prewarm_jit: bool = True`)**:
-  * Added automatic initialization-time Triton JIT compilation (`_prewarm_all_configs`) using 64-element dummy buffers (`128–256 bytes` VRAM) with `lr=0.0` and `weight_decay=0.0` across both Step 1 (`pass1` + `pass2`) and Step 2+ (`fused_single_pass` + `post_step_ema`) paths with target `BLOCK_SIZE` specialization (`1024` or `2048`), eliminating first-step compilation stalls.
-- **Bounded Bucket Chunking & Sequential In-Place Migration (`bucket_cap_mb: float = 500.0`)**:
-  * Bound contiguous bucket coalescing by an upper memory capacity ceiling $C_{\text{cap}} = 500\text{ MB}$, partitioning model parameters into ordered contiguous chunks $\mathcal{B}^{(1)}, \mathcal{B}^{(2)}, \dots, \mathcal{B}^{(K)}$ such that $\sum_{p \in \mathcal{B}^{(k)}} \text{bytes}(p) \le C_{\text{cap}}$ (with giant single layers $> C_{\text{cap}}$ isolated into individual buckets).
-  * Enforced sequential in-place migration with immediate parameter storage recycling (`del old_data`, `del old_grad`) and garbage collection, constraining peak transient memory allocation spikes during optimizer initialization to at most $\min(\text{model bytes}, C _ {\text{cap}})$.
-
----
-
-## [1.5.4] - 2026-10-29
-
-### Added
-- **In-Container Native Bitwise-BF16 State Promotion (`dtype_mode == 2`, `master_weights="none"`)**:
-  * Promoted 16-bit optimizer state buffers ($m_t$ via stochastic rounding `f32_to_bf16_bits_sr` and $v_t$ via round-to-nearest-even `f32_to_bf16_bits_rne`) to native Bitwise-BF16 stored inside the existing 16-bit container during pure `FP16` execution, expanding state exponent dynamic range to 8 bits ($1.18 \times 10^{-38}$) with zero additional VRAM overhead.
-
-### Changed
-- **Unified Denominator Stability Constant ($\epsilon_{\text{eff}} = \epsilon = 10^{-8}$)**:
-  * Unified the effective denominator constant $\epsilon_{\text{eff}} = \epsilon = 10^{-8}$ across all precision modes and hardware backends (replacing the legacy `FP16` floor clamp $\max(\epsilon, 10^{-4})$ introduced in `1.1.5`), preventing update velocity attenuation and late-stage convergence loss resets when $\sqrt{v_t} \ll 10^{-4}$.
-
-### Removed
-- **Centered Defect Compensation (`centered_defect`, $\mu_d$)**:
-  * Removed the `centered_defect` hyperparameter and the global defect mean reduction $\mu_d = \frac{1}{N}\sum_{i=1}^N (g_{t,i} - m_{t-1,i})$ across both Dual-Buffer Convex Lookahead Momentum (`buffer=2` / CLM) and Single-Buffer Triangle-Defect Convex Lookahead Momentum (`buffer=1` / TD-CLM).
-  * Simplified the instantaneous gradient-momentum residual to purely local coordinate-wise differences:
-
-    $\displaystyle d_t^{(\text{B1})} = g_t - m_{t-1}, \qquad d_t^{(\text{B2})} = g_t - m_t$
-
-  * Eliminated the pre-step global reduction pass (`_lumina_diff_sum_kernel` in Triton JIT/AOT, `diff_sum.comp` in Vulkan SPIR-V, `reduce_diff_sum_simd` in x86\_64 AVX-512/AVX2, and slot 5 of the 6-element `ScratchBuffer`), removing cross-lane/cross-rank reduction latency while retaining the 6-element physical `ScratchBuffer` layout (`[mask_sum, u_sq_sum, p_sq_sum, cautious_scale, bound_scale, _reserved]`, 5 active slots + 1 reserved slot for ABI alignment).
-
----
-
-## [1.5.3] - 2026-10-20
-
-No versions were published between 1.3.3 and 1.5.3. This release notes all changes made since version 1.3.3.
-
-### Added
-- **Single-Buffer Triangle-Defect Convex Lookahead Momentum (TD-CLM, `buffer=1`)**:
-  * Replaced the legacy 3-pass RMS single-buffer update with a single-pass formulation:
-
-    $\displaystyle \tilde{m} _ t = w_1 m _ {t-1} + w_2 g_t, \quad w_1 = \beta_1^2, \quad w_2 = 1 - \beta_1^2$
-
-    $\displaystyle \mathcal{E} _ t = w_1 |m _ {t-1}| + w_2 |g_t| + \lambda |d_t^{(\text{B1})}| + \epsilon _ {\text{eff}}$
-
-    $\displaystyle u_t^{(\text{B1})} = \frac{\tilde{m} _ t}{\mathcal{E} _ t} \in (-1, 1)$
-
-  * In 1.3.3, single-buffer mode required three sequential GPU kernel launches (`_lumina_v1_pass1_kernel`, `_lumina_v1_pass2_kernel`, `_lumina_v1_pass3_update_kernel`) and global atomic reductions across thread blocks to compute tensor-wide root-mean-square ($\text{rms} = \sqrt{\frac{1}{N}\sum \tilde{m} _ i^2 + \epsilon}$).
-  * TD-CLM replaces the tensor-wide reduction with coordinate-wise envelope bounding via the Triangle Inequality Envelope $\mathcal{E} _ t$. This reduces single-buffer execution to a single GPU pass and eliminates all $\tanh$ evaluations in the momentum path.
-  * Calibrated the defect penalty weight $\lambda$ to the symmetric spike-reversal equilibrium:
-
-    $\displaystyle \lambda^* = \frac{2(1 - \beta_1^2)^2}{4\beta_1^2 - 3}$
-
-    For the default $\beta_1 = 0.9$, $\lambda^* \approx 0.3008$, setting the default parameter to `lam=0.3`.
-
-- **Padé [13/6] Rational Approximation (`tanh_pade_f32`)**:
-  * Implemented a rational Padé [13/6] polynomial for hyperbolic tangent calculations:
-
-    $\displaystyle \tanh _ {\text{Pade}}(x) = \frac{x(1 + c_1 x^2 + c_2 x^4 + c_3 x^6 + c_4 x^8 + c_5 x^{10} + c_6 x^{12})}{1 + d_1 x^2 + d_2 x^4 + d_3 x^6}$
-
-  * Evaluated over the clamped domain $x \in [-7.9988117, 7.9988117]$ with asymptotic saturation to $\pm 1.0$.
-  * In 1.3.3, `_triton_tanh_fast` used hardware-dependent SFU instructions (`tl.math.tanh`) or analytical sigmoid fallbacks (`2 / (1 + exp(-2x)) - 1`). Different GPU architectures (e.g. Turing, Ampere, Hopper) evaluate SFU transcendental functions with slightly different mantissa rounding, leading to bitwise divergence during distributed training.
-  * Padé [13/6] provides uniform precision with $\lVert \tanh _ {\text{Pade}}(x) - \tanh(x) \rVert _ \infty < 1.5 \times 10^{-6}$ across CPU Scalar, AVX2, AVX-512, Vulkan SPIR-V, and Triton GPU kernels.
-
-- **Centered Innovation Defect (`centered_defect`)**:
-  * Added batch-mean centering for gradient innovation tracking (`centered_defect=False` by default):
-
-    $\displaystyle \mu_d = \frac{1}{N}\sum _ {i=1}^N (g _ {t,i} - m _ {t-1,i})$
-
-    $\displaystyle d_t^{(\text{B1})} = (g_t - m _ {t-1}) - \mu_d$
-
-    $\displaystyle d_t^{(\text{B2})} = (g_t - m_t) - \beta_1 \mu_d$
-
-  * In 1.3.3, uncentered innovation $g_t - m_t$ was used directly. On layers where gradients have a non-zero mean (such as normalization scales, biases, and token embeddings), the raw difference caused systematic drift in the second-moment accumulator. Centering eliminates this mean shift.
-
-- **Subnormal-Safe IEEE-754 FP16 Stochastic Rounding**:
-  * Added `f32_to_f16_bits_rne` to handle IEEE-754 subnormal numbers ($|x| < 2^{-14}$) during FP16 stochastic rounding.
-  * In 1.3.3, stochastic rounding added 13-bit random noise and masked with `& -8192`. When a floating-point value entered the subnormal range, bitwise addition directly corrupted the exponent field (bit 10), causing values to flip between normal and subnormal representations or trigger hardware flush-to-zero traps.
-  * The updated routine checks for zero exponent, shifts the significand into the subnormal format, and applies round-to-nearest-even tie-breaking before bit truncation.
-
-- **Multi-Backend Execution Engine**:
-  * Replaced the Python-only runtime with a multi-backend execution layer:
-    * **Rust SIMD (`luminav-simd`)**: AVX-512 (512-bit registers), AVX2 with FMA, portable scalar fallback, and Rayon parallel chunk scheduling (`ELEM_CHUNK_SIZE = 2,097,152`).
-    * **Vulkan UMA (`luminav-vulkan`)**: SPIR-V compute pipeline for cross-vendor GPUs (AMD, Intel, Apple Silicon via MoltenVK, and NVIDIA) with zero-copy host-visible memory buffers.
-    * **Triton AOT (`luminav-triton-aot`)**: C-ABI wrapper dispatching ahead-of-time precompiled kernels for 84 GPU architectures (Blackwell SM100/SM120, Hopper SM90, Ada SM89, Ampere SM80/SM86, Turing SM75, and AMD ROCm gfx90a/gfx942/gfx1100).
-    * **Standalone Runtime (`luminamoon.optimizer.standalone`)**: Pure NumPy and C-ABI execution that runs without requiring PyTorch.
-
-### Changed
-- **Parameter `alpha_ss`**: Removed. In 1.3.3, `alpha_ss=0.5` was used for soft-sign scaling $z / (1 + \alpha _ {\text{ss}} |z|)$ in the legacy single-buffer path. In 1.5.3, TD-CLM envelope normalization bounds updates directly to $(-1, 1)$, making soft-sign obsolete.
-- **Parameter `lam`**: Added with default `0.3`. Governs the defect penalty weight in TD-CLM.
-- **Parameter `centered_defect`**: Added with default `False`. Enables batch-mean centering of the innovation defect.
-- **Parameter `bound_type`**: Default changed from `"radial"` (1.3.3) to `"coordinate"` (1.5.3). Coordinate-wise hyperbolic bounding $(B_i \tanh(\Delta \theta_i / B_i))$ evaluates locally on each parameter without computing a tensor-wide norm reduction.
-- **Parameter `bound_ratio`**: Default changed from `0.03` to `0.05` to provide better update headroom on large model architectures.
-- **Parameter `ema_decay`**: Default changed from static `0.8` to `"auto"`. Resolves to `0.80` for Dual-Buffer mode (`buffer=2`) and `0.95` for Single-Buffer mode (`buffer=1`).
-- **Scratch Buffer**: Expanded from 5 floats `[mask_sum, u_sq_sum, p_sq_sum, cautious_scale, bound_scale]` to 6 floats, adding `diff_sum` at index 5 to accumulate batch-mean defect statistics.
-- **Cautious Scaling Floor**: Enforced `cautious_clamp_min=0.5` across all backends to cap maximum cautious energy scaling at $2.0\times$ ($2^1$).
-
-### Fixed
-- **Hardware Mantissa Divergence**: Fixed bitwise output differences across GPU families by replacing hardware SFU `tl.math.tanh` with Padé [13/6] rational approximation.
-- **FP16 Subnormal Bit Corruption**: Fixed exponent corruption in subnormal FP16 stochastic rounding by separating exponent extraction from significand shift logic.
-- **Single-Buffer Kernel Launch Latency**: Reduced single-buffer kernel dispatches from 3 passes to 1 pass, eliminating host-side synchronization bubbles and intermediate L2 atomic reduction overhead.
-- **Non-Zero Mean Gradient Drift**: Resolved directional accumulator drift on bias and normalization layers via `centered_defect`.
-
-### Parameter Comparison (1.3.3 vs 1.5.3)
-
-| Parameter | Version 1.3.3 | Version 1.5.3 | Technical Notes |
-| :--- | :--- | :--- | :--- |
-| `params` | `params` | `params` | Iterable of parameters or parameter groups. |
-| `lr` | `8e-4` | `8e-4` | Base learning rate. |
-| `betas` | `(0.9, 0.999)` | `(0.9, 0.999)` | First and second moment exponential decay rates. |
-| `eps` | `1e-8` | `1e-8` | Numerical stability floor. |
-| `weight_decay` | `8e-2` | `8e-2` | Decoupled weight decay rate. |
-| `tau` | `0.8` | `0.8` | Noise dispersion scale factor. |
-| `alpha_ss` | `0.5` | *(removed)* | Removed; superseded by TD-CLM envelope normalization. |
-| `lam` | *(none)* | `0.3` | Defect penalty weight in TD-CLM (calibrated to $\lambda^* \approx 0.3008$). |
-| `cautious` | `True` | `True` | Directional verification gate ($u_i g_i > 0$). |
-| `cautious_clamp_min` | `0.5` | `0.5` | Maximum cautious scale cap ($2.0\times$). |
-| `buffer` | `2` | `2` | `1` = Single-Buffer (TD-CLM), `2` = Dual-Buffer (CLM). |
-| `centered_defect` | *(none)* | `False` | Enables subtraction of batch-mean defect $\mu_d$. |
-| `stochastic_rounding` | `True` | `True` | Bitwise stochastic rounding on FP16/BF16 weights. |
-| `bound` | `True` | `True` | Enables update trust-region bounding. |
-| `bound_type` | `"radial"` | `"coordinate"` | Default changed to coordinate-wise bounding (zero reduction overhead). |
-| `bound_ratio` | `0.03` | `0.05` | Maximum step magnitude relative to parameter magnitude. |
-| `master_weights` | `False` | `"none"` | `"none"`, `"semi"`, or `"full"`. |
-| `execution` | `"auto"` | `"auto"` | Dynamic routing: SIMD, Vulkan, Triton, or Foreach. |
-| `fused_single_pass` | `False` | `True` | Enables single-pass streaming with EMA scaling. |
-| `ema_decay` | `0.8` | `"auto"` | Automatically sets `0.80` for dual-buffer, `0.95` for single-buffer. |
-| `fsdp_sync` | `False` | `False` | Cross-rank collective all-reduce synchronization. |
-| `seed` | `1337` | `1337` | PRNG seed for deterministic stochastic rounding. |
-
----
-
-## [1.3.3] - 2026-10-06
-
-### Fixed
-* **Triton AST Compiler Early-Return Inlining Error:**
-  * Resolved fatal MLIR/CFG code generation failure (`at 35:4: _store_state_buffer(...) ^`) triggered by mid-branch `return` statements in `@triton.jit` helper functions (`_store_param` and `_store_state_buffer`).
-  * Refactored helper control flow into structured, single-exit `if ... elif ... else` branches, ensuring compliant JIT function inlining across all kernel entry points.
-* **Bitcast Width Mismatch (Int64 Promotion in Stochastic Rounding):**
-  * Fixed implicit 64-bit integer promotion caused by Python integer literals (`0x9E3779B9`, `~0x1FFF`, `~0xFFFF`) inside Triton bitwise operations.
-  * Enforced strict 32-bit signed arithmetic via two's complement constants (`-1640531527`, `-2048173461`, `-1028477387`) with explicit `.to(tl.int32)` coercion, and standardized bitwise masks (`-65536` for BF16, `-8192` for FP16), ensuring type-safe 32-bit bitcasting (`tl.int32` to `tl.float32`).
-* **JIT Kernel Call Keyword Arguments:**
-  * Replaced internal keyword argument invocations (`use_sr=False`) with standard positional boolean parameters inside Triton JIT kernels to eliminate compiler argument binding warnings across Triton runtime versions.
-
-### Improved
-* **Throttled Fallback Telemetry & Console De-Noising:**
-  * Added `self._warned_triton_failure` state tracking to prevent multi-line compiler AST dumps from flooding the console across every parameter tensor in a parameter group.
-  * Formatted fallback notifications into a single concise one-line `WARNING` log on initial encounter, routing full tracebacks to `DEBUG` level.
-
-### Verified
-* **End-to-End JIT Execution on NVIDIA Architectures (Tesla T4):**
-  * Validated JIT execution across both Dual-Pass (`fused_single_pass=False`) and Fused Single-Pass (`fused_single_pass=True`) modes in native FP16.
-  * Confirmed non-zero weight update dynamics ($|\Delta W| > 0$) without gradient underflow or weight freezing in pure master-free training for Version 1.3.3.
-
----
-
-## [1.3.2] - 2026-10-06
-
-### Added
-- **Distributed FSDP Collective Synchronization (`fsdp_sync`)**:
-  - Introduced the `fsdp_sync: bool = False` configuration flag on optimizer initialization, backed by automated runtime guarding via `torch.distributed.is_initialized()`.
-  - Integrated cross-GPU collective communications (`torch.distributed.all_reduce(..., op=torch.distributed.ReduceOp.SUM)`) for global reduction metrics (`mask_sum`, `u_sq_sum`, and `p_sq_sum`) across Triton kernels, single-step execution, and vectorized foreach loops.
-  - Maintains bit-exact scalar consistency for `m_bar`, `cautious_scale`, and `bound_scale` across distributed parameter shards during large-scale pretraining under PyTorch FSDP (Fully Sharded Data Parallel) and DeepSpeed ZeRO-3.
-- **Dedicated Low-Precision Stochastic Quantization Routine (`_sr_quantize`)**:
-  - Factored out bitwise stochastic rounding into a dedicated, modular helper function: `_sr_quantize(target, val_f32, use_sr, generator)`.
-  - Enabled native stochastic rounding on half-precision momentum states (`exp_avg` in BF16/FP16) across all PyTorch CPU/CUDA fallback paths (`_single_step_v2`, `_foreach_step_v2`, `_single_step_v1`, and `_foreach_step_v1`), systematically eliminating gradient stagnation induced by fractional bit truncation (LSB underflow).
-- **Persistent Contiguous Buffer Caching**:
-  - Introduced persistent state cache tensors within `self.state[p]` (`p_contig`, `exp_avg_contig`, `exp_avg_sq_contig`, and `mp_contig`) to stage strided or sliced parameters before kernel launch.
-  - Eliminates dynamic memory allocations (`.contiguous()`) during iterative training loops, preserving PyTorch VRAM caching allocator stability and preventing CUDA Graph capture invalidation.
-- **Zero-Copy Direct Tensor Aliasing for FP32 Master Weights**:
-  - When parameter tensors are natively `torch.float32`, `state["master_param"]` now directly aliases the parameter tensor pointer (`state["master_param"] = p`), avoiding additional master weight VRAM overhead for FP32 layers (0 bytes auxiliary memory footprint).
-- **Checkpoint Sanitizer via State Dict Filter (`state_dict`)**:
-  - Overrode `state_dict()` to automatically purge temporary contiguous cache buffers (`p_contig`, `exp_avg_contig`, `exp_avg_sq_contig`, `mp_contig`), ensuring on-disk checkpoint payloads remain clean, compact, and fully backward-compatible.
-- **Defensive Device Pipeline Flush on Kernel Fault (`torch.cuda.synchronize`)**:
-  - Embedded explicit hardware device synchronization (`torch.cuda.synchronize(p.device)`) within Triton exception recovery handlers prior to PyTorch fallback execution, preventing asynchronous CUDA error cascades into subsequent tensor operations.
-
-### Changed
-- **High-Entropy Bit-Avalanche PRNG Hash Function (`_prng`)**:
-  - Replaced the standard Linear Congruential Generator (LCG) with a MurmurHash3 / SplitMix32 integer bit-avalanche hashing routine:
-    ```python
-    z = idx + seed + 0x9E3779B9
-    z = (z ^ (z >> 16)) * 0x85EBCA6B
-    z = (z ^ (z >> 13)) * 0xC2B2AE35
-    z = z ^ (z >> 16)
-    ```
-  - Delivers uniform pseudo-random noise distributions with zero low-order bit periodicity, significantly refining stochastic rounding fidelity on BF16/FP16 parameters.
-- **Memory-Contiguous Scratch Buffer Realignment for v1 Triton Kernels**:
-  - Re-indexed reduction buffer offsets in Triton v1 kernels:
-    - Pass 1 metrics (`[rms_sum, p_sq_sum]`) mapped to contiguous indices `[0, 1]`.
-    - Pass 2 metrics (`[mask_sum, u_sq_sum]`) mapped to contiguous indices `[2, 3]`.
-  - Enables vectorized contiguous slice all-reduces (`scratch_param[:2]` and `scratch_param[2:4]`) under FSDP without requiring non-contiguous memory gather/scatter overhead.
-- **Multi-GPU Device Index Resolution in RNG Dispatcher (`_get_generator`)**:
-  - Fixed device index resolution when `device` is passed as a generic `torch.device("cuda")` without an explicit index by defaulting to `torch.cuda.current_device()`, avoiding cross-GPU generator key collisions during multi-GPU distributed runs.
-- **Conditional Atomic Reduction Pruning in v2 Pass 1 Kernel**:
-  - Guarded `mask_sum` atomic accumulation in `_lumina_v2_pass1_kernel` with `if cautious:`, bypassing CTA reductions and L2 atomic additions when `cautious=False`.
-- **Expanded Master Weight Precision Configuration Aliases**:
-  - Added the `"half"` alias for `"semi"` mode alongside `"fp32"` and `"masterweight"` aliases for `"full"` mode.
-- **Metadata & Documentation Header Update**:
-  - Bumped specification version to `1.3.2`.
-
-### Fixed
-- **Singular Zero-Division & Step-Zero Bias Correction (`safe_base_step`)**:
-  - Introduced `safe_base_step = max(base_step, 1)` and `bc2_base = max(1.0 - (beta2 ** safe_base_step), 1e-15)` across Triton, single-step, and foreach loops.
-  - Fully resolves potential `ZeroDivisionError` and NaN blowups in `c1` / `c2` computations when steps evaluate at `step == 0`.
-- **Numerical Overflow Clamp in Sigmoid-Based Tanh Fallback**:
-  - Applied register clamp bounds `x_clamped = tl.maximum(tl.minimum(x, 20.0), -20.0)` in analytical fallback `_triton_tanh_fast`, preventing `tl.exp(-2.0 * x)` from overflowing to `inf` or `nan` on devices lacking hardware SFU tanh instructions.
-- **Missing Argument Forwarding in PyTorch Single-Step v1**:
-  - Corrected `_single_step_v1` method signature and call-sites to accept and propagate `scratches`, eliminating runtime `TypeError` during fallback invocation.
-- **Multi-Rank Synchronization Skew in Sharded Parameter Bounding**:
-  - Aligned tensor norms and radial bounding factors across sharded ranks via synchronized global scalar reductions, preventing parameter update drift across distributed ranks.
-
-### Performance & Engine Refactoring
-- **Zero Dynamic VRAM Allocation on Strided / Sliced Tensors**:
-  - Persistent pre-allocated contiguous staging buffers avoid PyTorch caching allocator churn and eliminate CUDA Graph memory address invalidations.
-  - Preserves deterministic memory footprints across long pre-training runs.
-- **Optimized Distributed Collective Bandwidth**:
-  - Grouping scalar reductions into contiguous vector slices compresses distributed synchronization overhead into a single collective kernel launch per pass.
-
----
-
-## [1.3.0] - 2026-09-23
-
-### Added
-- **Fused Single-Pass Kernel with Post-Step EMA (`fused_single_pass`, `ema_decay`)**:
-  - Introduced `_lumina_v2_fused_single_pass_kernel` coupled with an asynchronous CTA micro-kernel `_lumina_v2_post_step_ema_kernel[(1,)]`.
-  - When `fused_single_pass=True`, avoids two-pass global reductions by leveraging running Exponential Moving Average (EMA) estimates of `cautious_scale` ($1 / \bar{M}$) and `bound_scale` ($\tanh(r) / r$).
-  - Unifies gradient filtering, momentum accumulation, variance dispersion, cautious masking, radial bounding, decoupled weight decay, stochastic rounding, and block partial reductions into a **single unified VRAM-resident pass**, reducing global GPU memory bandwidth traffic per optimizer step.
-- **Hybrid Master Weight Precision Architecture (`master_weights`)**:
-  - Added multi-tier master weight flexibility via `master_weights: Union[bool, str] = False`:
-    - `"none"` / `False` *(Default)*: Pure master-free low-precision training leveraging on-chip bitwise stochastic rounding directly on native half-precision weights.
-    - `"semi"` / `"hybrid"` / `"half"`: 32-bit FP32 master parameters (`state["master_param"]`) coupled with 16-bit low-memory optimizer states (`exp_avg`, `exp_avg_sq` in BF16/FP16), saving ~50% optimizer state VRAM compared to traditional FP32 setups while maintaining full FP32 weight accumulation accuracy.
-    - `"full"` / `"fp32"` / `"masterweight"`: Traditional full FP32 master parameters with full FP32 optimizer states for standard baseline comparisons.
-- **In-Kernel In-Flight NaN / Inf Gradient Sanitization**:
-  - Embedded register-level defensive sanitization across all Triton GPU kernels: `is_finite_g = (g == g) & (g < 1e38) & (g > -1e38); g = tl.where(is_finite_g, g, 0.0)`.
-  - Replaces non-finite gradients and subnormal corruptions with zero in registers before they can contaminate momentum (`exp_avg`) or variance (`exp_avg_sq`) state buffers.
-- **Low-Precision State Buffer Serialization (`_store_state_buffer`)**:
-  - Added a dedicated Triton state serialization routine supporting FP32, BF16, and FP16 state representations with optional bitwise stochastic rounding applied directly to momentum buffers.
-- **Multi-Device & Cross-Accelerator Hardware Context Isolation (`_get_device_context`)**:
-  - Unified device stream scoping supporting NVIDIA CUDA (`torch.cuda`), Intel XPU (`torch.xpu`), and modern PyTorch 2.x unified accelerator interfaces (`torch.accelerator.device`), eliminating multi-GPU stream hijacking and cross-device race conditions.
-- **Strict Determinism Auto-Routing**:
-  - Automatically queries `torch.are_deterministic_algorithms_enabled()`. When enabled under `execution="auto"`, dynamically routes execution to C++ vectorized `foreach` to preserve bitwise cross-run reproducibility, circumventing non-deterministic floating-point hardware atomic reordering.
-- **Granular Fault Recovery with State Guarding (`completed_indices`)**:
-  - Implemented per-parameter completion tracking in `_dispatch_triton`. If a kernel encounters an unexpected runtime fault, completed parameters are preserved while uncompleted ones fall back cleanly to `foreach`.
-  - Added `_single_step_v2_apply_param` and `_single_step_v1_apply_param` to safely finalize parameter updates if Pass 1 had already committed state transitions, strictly preventing double-update state corruption.
-
-### Changed
-- **Per-Parameter Persistent Scratch Tensors (Elimination of Global Slicing)**:
-  - Replaced the global dynamic slice allocator (`self._scratch_tensors[dev][:count]`) with isolated per-parameter persistent state allocations (`state["scratch"] = torch.zeros((5,), device=p.device, dtype=torch.float32)`).
-  - Memory pointers for `[mask_sum, u_sq_sum, p_sq_sum, cautious_scale, bound_scale]` remain completely invariant throughout the entire training lifecycle, preserving pointer stability for **CUDA Graph Capture** (`torch.accelerator.Graph`) and TorchInductor AOT compilation without memory aliasing.
-- **Analytically Exact Nesterov Bias Correction ($\beta_1^{\text{step} + 1}$)**:
-  - Corrected first-moment bias correction from the classical Adam formulation ($1.0 - \beta_1^{\text{step}}$) to the exact analytical expectation of the Nesterov blended momentum: `bc1_nes = 1.0 - beta1^(step + 1)`. Removes residual momentum bias during early warmup iterations.
-- **Adaptive CTA Tile Sizing and Dynamic Thread Warp Scaling**:
-  - Introduced hardware-aware CTA tile scaling based on parameter volume: parameters with $\ge 2,097,152$ elements (2M+) dynamically configure `BLOCK_SIZE = 2048` and `num_warps = 8` (versus `BLOCK_SIZE = 1024`, `num_warps = 4` for smaller tensors).
-  - Cuts total thread-block grid counts and L2 cache atomic addition pressure by 50% on large MLP, Attention Projection, and Vocabulary layers.
-- **Hardware-Native Tanh Instruction Dispatch (`tl.math.tanh` / `tl.tanh`)**:
-  - Upgraded `_triton_tanh_fast` to dynamically inspect compiler capabilities, binding directly to native PTX Special Function Unit (SFU) instructions (`tanh.approx.f32`) on modern Triton compilers while retaining analytical sigmoid fallbacks for legacy environments.
-- **PRNG Decorrelation between Momentum and Parameter Updates**:
-  - Separated RNG seed streams using bitwise alternating mask shifts: `seed_p = (seed + 0x55555555) & 0x7FFFFFFF`, mathematically eliminating cross-coupling noise correlation between state buffer rounding and parameter update rounding.
-- **Bumped specification version metadata to 1.3.0** across `luminav.py`.
-
-### Fixed
-- **Floating-Point Cancellation in Radial Bounding ($r \to 0$)**:
-  - Fixed precision jitter and roundoff degradation near zero by adding an asymptotic Taylor-series guard across Triton and PyTorch bounding engines: `torch.where(r < 1e-4, 1.0, tanh(r) / r)`.
-- **Artificial Denominator Dilation in Dispersion Scale ($\sigma$)**:
-  - Replaced additive epsilon inflation (`sigma + 1e-6`) with a non-distorting floor clamp (`tl.maximum(sigma, 1e-12)`), preventing artificial dampening of subtle gradient signals.
-- **Singular Zero-Division in Parameter Norm Clamping**:
-  - Replaced `norm_p + 1e-6` with `torch.clamp_min(norm_p, 1.0)` / `tl.maximum(norm_p, 1.0)`, maintaining numerical stability even when parameters are initialized near zero.
-- **Transient Memory Pool Spikes during PyTorch Fallback Stochastic Rounding**:
-  - Refactored `_sr_update` to stream stochastic noise generation in bounded chunks of 1,048,576 elements (~4 MB chunk size), eliminating multi-hundred megabyte temporary int32 tensor spikes during fallback execution on large parameter matrices.
-- **Non-Contiguous Buffer Safety & Strided Memory Copy**:
-  - Added strict contiguous layout verification for parameters, states, and master weights with in-place synchronization (`p.copy_(p_contig)`), guarded by a one-time developer warning.
-
-### Performance & Engine Refactoring
-- **~50% Global Memory Bandwidth Reduction via Fused Single-Pass Mode**:
-  - By unifying parameter loading, gradient evaluation, state transformation, and bounding into one GPU pass under `fused_single_pass=True`, VRAM read/write cycles are halved for cautious and radially bounded steps.
-- **Sub-Microsecond L2 Hardware Atomic Reduction Architecture**:
-  - Formally documented the hybrid Intra-Block Tree Reduction + Inter-Block Hardware Atomic pattern. With `BLOCK_SIZE=2048`, a 10M-parameter tensor issues only 4,883 atomic additions directly into GPU L2 cache crossbars, executing in < 0.5 µs and strictly outperforming multi-tier cascaded reduction micro-kernels by eliminating 3–5 µs driver dispatch bubbles.
-- **Static Buffer Address Invariance for CUDA Graphs & TorchInductor AOT**:
-  - Persistent 5-slot state scratch buffers eliminate all dynamic slicing and memory reallocations during optimization loops, preserving pointer address invariance for CUDA Graph replay.
-
----
-
-## [1.2.1] - 2026-09-17
-
-### Added
-- **In-Kernel GPU Parameter Norm Reduction (`p_sq_sum_ptr`)**: Fused parameter Euclidean norm squared $\sum(p_i^2)$ calculation directly into Pass 1 Triton reduction kernels (`_lumina_v2_pass1_kernel` and `_lumina_v1_pass1_kernel`). Pass 2 and Pass 3 now load and resolve `norm_p` (`tl.sqrt(tl.maximum(p_sq_sum, 0.0))`) entirely inside GPU SRAM/registers without host intervention.
-- **Stateless CPU Integer Multiplicative PRNG Hashing**: Replaced dynamic GPU RNG tensor allocation with a pure CPU integer hash using Knuth's 32-bit golden ratio constant: `(step * 0x9E3779B9 + i * 10007) & 0x7FFFFFFF`. Generates uncorrelated 31-bit integer seeds per layer with zero GPU kernel launch overhead and zero PCIe host-device latency.
-- **Isolated Generator Cache for Fallback Engine (`self._generators`)**: Added an internal per-device `torch.Generator` cache, ensuring stochastic rounding in `_sr_update` operates in complete isolation from PyTorch's default CUDA generator state.
-
-### Changed
-- **Consolidated Scratch Buffer Zeroing**: Removed redundant per-parameter slice zeroing inside parameter loops. Buffer zeroing is now batched into a single consolidated `scratch.zero_()` call per step inside `_get_scratch_buffer`.
-- **Scratch Buffer Stride Expansion**:
-  - `buffer=2`: Expanded stride from 2×N to 3×N elements per parameter to host `[mask_sum, u_sq_sum, p_sq_sum]`.
-  - `buffer=1`: Expanded stride from 3×N to 4×N elements per parameter to host `[rms_sum, mask_sum, u_sq_sum, p_sq_sum]`.
-- **Bumped specification version metadata to 1.2.1**.
-
-### Fixed
-- **Host-Device Synchronization & Pipeline Stalls (`.item()` Bottleneck)**: Removed `norm_p = float(p_contig.float().norm().item())` and `seed = torch.randint(...).item()` from Triton step pipelines. Eliminated 200–400 synchronous PCIe roundtrips and GPU pipeline stalls per step on deep architectures (e.g., 28-layer Transformers), restoring full asynchronous GPU compute utilization.
-- **CUDA Graphs & `torch.compile` Graph Breaks**: Fixed fatal graph capture crashes and TorchDynamo graph breaks caused by blocking host-device synchronization during radial bounding and seed generation. The optimizer step is now compatible with CUDA Graphs (`torch.accelerator.Graph`) and TorchInductor compilation.
-- **Transient VRAM Allocator Thrashing**: Eliminated repeated temporary FP32 parameter cloning (`p_contig.float()`) during norm evaluation, preventing memory pool fragmentation in tight VRAM regimes.
-- **Global RNG Pollution & Determinism Compliance**: Fixed global RNG state mutation in fallback stochastic rounding. Solved reproducibility drift in upstream Dropout, DataLoader shuffling, and RLHF reward modeling, ensuring strict adherence to PyTorch 2.10+ `torch.use_deterministic_algorithms(True)`.
-
-### Performance & Engine Refactoring
-- **Batch Kernel Launch Overhead Elimination**: Removed up to 400–800 individual `cudaMemsetAsync` micro-calls per step previously triggered by per-slice `.zero_()` invocations on 1-element scratch tensors.
-- **Single-Pass Bypass for Coordinate Bounding**: Streamlined `_triton_step_v2` to immediately bypass two-pass reductions and invoke `_lumina_v2_single_pass_kernel` when `cautious=False` and coordinate bounding (`bound_mode == 2`) is selected.
-
----
-
-## [1.2.0] - 2026-09-17
-
-### Added
-- **Automatic Dynamic FP16 Cliff Governor:** Built-in asymptotic boundary governor (`_triton_tanh_fast` in GPU register and PyTorch fallback) that automatically activates when `p.dtype == torch.float16`. Prevents unclipped steps from pushing parameters, activations, and residual streams into the IEEE-754 FP16 overflow cliff (> 65,504). Remains completely dormant (zero overhead) on `bfloat16` (TPU / Ampere+) and `float32`.
-- **Smooth Asymptotic Parameter Bounding (`bound`, `bound_type`, `bound_ratio`):** Added non-intrusive step bounding with two mathematically continuous formulations:
-  - `"radial"`: Direction-Preserving Radial Squashing (`update * tanh(r) / r`) preserving gradient direction (Cosine Similarity = 1.000).
-  - `"coordinate"`: Dual-Hyperbolic Coordinate Squashing for fine-grained per-element damping.
-  - Configurable via constructor: `bound: bool = True`, `bound_type: str = "radial"`, `bound_ratio: float = 0.03`.
-
-### Changed
-- **Increased Default `cautious_clamp_min` from `0.2` to `0.5`:** Enforces an exact power-of-two maximum energy scaling factor (2.00x, $2^1$). Eliminates floating-point mantissa roundoff distortion in IEEE-754 hardware.
-- Bumped specification version metadata to `1.2.0`.
-
-### Fixed
-* **FP16 Step Divergence & NaN Overflow at High Learning Rates:** Solved step divergence and subsequent NaN crashes when training deep 28-layer Transformer architectures (e.g., Qwen3-0.6B) in pure FP16 at nominal learning rates (`lr = 8e-4`).
-* **Sparse Embedding & LM-Head Amplification:** Mitigated 5x over-amplification on sparse vocabulary gradients where active token counts are much smaller than total tensor parameters ($N \gg N_{\text{active}}$).
-* **Verified stable 1,000-step pretraining from scratch on Qwen3-0.6B** (seq length 512, pure FP16, `lr = 8e-4`) on Nvidia Tesla T4 with zero gradient clipping and zero LR scheduler (loss dropped from 12.09 down to ~5.7 with zero NaNs).
-* **Weight Decay Numerical Parity on Fallback Engine:** Resolved a silent precision-stagnation issue in `_single_step` and `_foreach_step`. Previously, decoupled weight decay (`p.mul_(1.0 - lr * weight_decay)`) was executed directly on native FP16/BF16 tensors prior to `_sr_update`. Due to IEEE-754 Round-to-Nearest (RTN) limits, `(1 - lr * weight_decay) ≈ 0.999936` rounded back to `1.0`, causing weight decay to freeze on half-precision parameters. Weight decay is now unified into FP32 registers inside `_sr_update` alongside stochastic rounding, matching the numerical behavior with the Triton kernel.
-* **Fallback Foreach Indentation & Scope Alignment:** Corrected tensor iteration scope and indentation for `torch._foreach_add_` in non-SR fallback branches.
-* **In-Place GPU Clamping in `_apply_bound`:** Replaced dynamic GPU scalar allocations (`torch.tensor(1.5e-4, device=p.device)`) with native `torch.clamp_min`, eliminating transient CUDA malloc spikes per update step.
-
-### Performance & Engine Refactoring
-* **Unified Bound Execution Pipeline:** Added an immediate short-circuit guard (`if lr == 0.0: return update`) and streamlined the execution flow: intermediate updates are scaled in FP32 (`step_val = lr * u_float`), passed through radial/coordinate bounding, chained cleanly into the FP16 Cliff Governor, and returned in a single normalized step.
-
----
-
-## [1.1.5] - 2026-09-15
-
-### Added
-- **Native Hardware Capability Detection (`_supports_native_bf16`):** Added automatic device capability detection via `torch.cuda.is_bf16_supported()` to identify Ampere SM80+ architectures and streamline native execution paths.
-- **Hugging Face Repository Reference:** Added official Hugging Face repository link in module docstrings.
-
-### Changed
-- **Dynamic Epsilon Clamping for FP16:** Dynamically floors effective epsilon to `max(eps, 1e-4)` when parameters or states are in `torch.float16`. Prevents bias-corrected scale factor `c2` from underflowing into IEEE-754 subnormal/zero limits on step 1.
-- Bumped specification version to `1.1.5`.
-
-### Fixed
-- **FP16 Step-1 `NaN` from Epsilon Underflow:** Fixed `0.0 / 0.0 = NaN` division occurring on initial training steps when parameters receive zero gradients (e.g., unselected vocabulary embeddings, padding tokens, or dropout paths).
-- **Second-Moment Variance Saturation (`+inf`):** Added safe upper-bound register clamping (`tl.minimum(v_new, 65000.0)` in Triton kernels and `.clamp_(0.0, 65000.0)` in PyTorch C++ foreach/single loops). Prevents unclipped initial pretraining gradient surges ($|g| > 256$) from overflowing the FP16 maximum dynamic range (65,504) into irreversible `+inf`.
-- **Zero-Sigma Division Guard:** Injected explicit `sigma + 1e-6` denominator guards across all Triton kernel passes and PyTorch update steps to ensure numerical safety during cold-start iterations.
-- Verified stable 100-step pretraining from scratch on `SmolLM-135M` using Wikimedia Wikipedia in pure FP16 on Nvidia Tesla T4 GPU with zero gradient clipping.
-
----
-
-## [1.1.0] - 2026-09-11
-
-### Added
-- **Bitwise Stochastic Rounding (SR):** Integrated on-chip PRNG integer bit-manipulation hash in Triton kernels and vectorized PyTorch fallback for native FP16/BF16 training without master weights.
-- Unified Triton store routine introducing compile-time branching (`dtype_mode` / `use_sr: tl.constexpr`) with explicit safe casting for FP16 and BF16 pointers.
-- Vectorized fallback improvements utilizing C++ multi-tensor operations when stochastic rounding is disabled or tensors are FP32.
-- State buffer contiguity validation (`exp_avg_contig`, `exp_avg_sq_contig`) ensuring safe linear memory addressing before kernel dispatch.
-- Declarative configuration file (`config.json`) defining hyperparameters and optimizer presets.
-
-### Changed
-- Refactored PRNG integer arithmetic to strictly stay within signed 32-bit integer limits ($< 2^{31} - 1$), eliminating JIT compilation failures on newer Triton releases.
-- State buffers (`exp_avg`, `exp_avg_sq`) now initialize with strict contiguous memory format.
-
-### Fixed
-- Fixed `ValueError: Scalar out of range for type int32` and Triton LLVM compile errors during unsigned 32-bit bitmask operations.
-- Resolved kernel fallback hangs and compilation overhead on CUDA devices.
-
----
-
-## [1.0.2] - 2026-09-08
-
-### Added
-- Official Apache-2.0 license notice and copyright headers.
-- Standardized package exports (`__all__`, `__version__`, `__author__`, `__license__`).
-
----
-
-## [1.0.1] - 2026-08-31
-
-### Fixed
-- **Memory Synchronization Fix (`423d171`):** Fixed a critical bug where non-contiguous parameter tensors (e.g., transposed linear layers or sliced weights) were updated on temporary contiguous copies without reflecting changes back to model weights. Added `p.copy_(p_contig)` after Triton execution.
-
----
-
-## [1.0.0] - 2026-08-30
-
-### Added
-- Initial release of the LuminaV optimizer.
-- **Zero Master-Weight Copies:** Direct parameter updates in native half-precision without allocating 4-byte FP32 master weights.
-- **Hyperbolic Tangent (`tanh`) Bounding Envelope:** Projects normalized momentum ratio into `(-1.0, 1.0)` to eliminate exploding gradient shocks.
-- **Directional Alignment Gate:** Cautious update masking that zeroes out momentum steps opposing mini-batch gradient directions.
-- **Central Innovation Variance:** Noise dispersion tracking $(g_t - m_t)^2$ decoupling mini-batch turbulence from directional descent.
-- Dual-buffer mode (`buffer=2`, LuminaV-2B) and single-buffer mode (`buffer=1`, LuminaV-1).
-- Custom fused OpenAI Triton kernels with automated fallback execution.
+# LuminaV Optimizer Algorithm Changelog
+
+All notable changes to the LuminaV Optimizer algorithm are documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+* [Software Package Changelog (`luminamoon`)](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/changelogs/SOFTWARE_CHANGELOG.md)
+
+---
+
+## [1.6.0] - 2026-11-03
+
+### Added
+- **T-Family (Triangle Family) tanh-free Modes (`buffer="T1"` & `buffer="T2"`)**:
+  * **`buffer="T2"` (2-Buffer Triangle Family Mode)**: Keeps the 2-buffer state (`exp_avg` and `exp_avg_sq`) without using $\tanh$, computing the gradient-momentum residual $d_t = g_t - m _ {t-1}$ and bounding the lookahead update $\tilde{m}_t$ via the **Triangle Inequality Envelope** $\mathcal{E}_t^{(\text{T2})}$:
+
+    $\displaystyle d_t = g_t - m _ {t-1}, \qquad v_t = \beta_2 v _ {t-1} + (1 - \beta_2) d_t^2$
+
+    $\displaystyle \tilde{m} _ t = w_1 m _ {t-1} + w_2 g_t, \qquad w_1 = \beta_1^2, \qquad w_2 = 1 - \beta_1^2$
+
+    $\displaystyle \mathcal{E} _ t^{(\text{T2})} = w_1 |m _ {t-1}| + w_2 |g_t| + 0.5\lambda |d_t| + 0.5\lambda \sigma_t + \epsilon _ {\text{eff}}, \qquad u_t^{(\text{T2})} = \frac{\tilde{m} _ t}{\mathcal{E} _ t^{(\text{T2})}} \in (-1, 1)$
+
+    By the Triangle Inequality ($|w_1 m _ {t-1} + w_2 g_t| \le w_1 |m _ {t-1}| + w_2 |g_t| \le \mathcal{E} _ t^{(\text{T2})}$), $|u_t^{(\text{T2})}| < 1$ is guaranteed without evaluating $\tanh$. The term $0.5\lambda |d_t|$ acts as a **triangle-defect penalty**, and $0.5\lambda \sigma_t$ provides belief-variance damping.
+  * **`buffer="T1"` (1-Buffer Triangle Family Mode)**: Combines the single-buffer **Triangle-Defect** denominator $\mathcal{E} _ t^{(\text{T1})} = w_1 |m _ {t-1}| + w_2 |g_t| + \lambda |d_t| + \epsilon _ {\text{eff}}$ (using residual $d_t = g_t - m _ {t-1}$) with the rational update bounds below, reducing optimizer state memory by 50% via the Triangle Inequality.
+  * **Rational Update Bounds (for `T1` & `T2`)**: Replaces $\tanh$ and $\sqrt{\cdot}$ in the coordinate, radial, and FP16 update bounds with simple rational formulas:
+
+    $\displaystyle b_i = \mathrm{bound} _ {\text{ratio}} \cdot (|w_i| + 10^{-2}), \qquad \Delta w_i = \frac{b_i (b_i + |s_i|) s_i}{b_i (b_i + |s_i|) + s_i^2}, \qquad \mathrm{scale} _ {\text{radial}} = \frac{1 + r}{1 + r + r^2}$
+  * Keeps 100% backward compatibility and identical outputs for existing modes (`buffer=1`, `buffer=2`, and `version="1.5.5"`).
+
+---
+
+## [1.5.5] - 2026-10-29
+
+### Added
+- **Contiguous Bucket Fusion (`fuse_tensors: bool = False`)**:
+  * Added opt-in contiguous parameter and state coalescing (`_build_contiguous_buckets`) that groups parameters within each `(param_group, device, dtype)` domain $\mathcal{B}_{g,d,\tau}$ into contiguous 1D storage (`flat_p`, `flat_g`, `flat_m`, `flat_v`, `flat_mp`) and rebinds `p.data`, `p.grad`, and `self.state[p]` entries as zero-copy views (`flat_*[offset : offset + n].view(p.shape)`).
+  * Eliminates per-tensor CPU-to-GPU kernel launch latency on fragmented vision and edge architectures by executing an entire bucket in a single fused kernel dispatch while computing `cautious_scale` and `radial` trust bounds across the bucket domain $\mathcal{B}_{g,d,\tau}$.
+  * Added in-place `zero_grad()` view preservation (`flat_g.zero_()`) and full `state_dict()` / `load_state_dict()` checkpoint compatibility without leaking temporary buffer keys (LSO-Eval Stage II compliant).
+- **Triton JIT Pre-Warmup (`prewarm_jit: bool = True`)**:
+  * Added automatic initialization-time Triton JIT compilation (`_prewarm_all_configs`) using 64-element dummy buffers (`128–256 bytes` VRAM) with `lr=0.0` and `weight_decay=0.0` across both Step 1 (`pass1` + `pass2`) and Step 2+ (`fused_single_pass` + `post_step_ema`) paths with target `BLOCK_SIZE` specialization (`1024` or `2048`), eliminating first-step compilation stalls.
+- **Bounded Bucket Chunking & Sequential In-Place Migration (`bucket_cap_mb: float = 500.0`)**:
+  * Bound contiguous bucket coalescing by an upper memory capacity ceiling $C_{\text{cap}} = 500\text{ MB}$, partitioning model parameters into ordered contiguous chunks $\mathcal{B}^{(1)}, \mathcal{B}^{(2)}, \dots, \mathcal{B}^{(K)}$ such that $\sum_{p \in \mathcal{B}^{(k)}} \text{bytes}(p) \le C_{\text{cap}}$ (with giant single layers $> C_{\text{cap}}$ isolated into individual buckets).
+  * Enforced sequential in-place migration with immediate parameter storage recycling (`del old_data`, `del old_grad`) and garbage collection, constraining peak transient memory allocation spikes during optimizer initialization to at most $\min(\text{model bytes}, C _ {\text{cap}})$.
+
+---
+
+## [1.5.4] - 2026-10-29
+
+### Added
+- **In-Container Native Bitwise-BF16 State Promotion (`dtype_mode == 2`, `master_weights="none"`)**:
+  * Promoted 16-bit optimizer state buffers ($m_t$ via stochastic rounding `f32_to_bf16_bits_sr` and $v_t$ via round-to-nearest-even `f32_to_bf16_bits_rne`) to native Bitwise-BF16 stored inside the existing 16-bit container during pure `FP16` execution, expanding state exponent dynamic range to 8 bits ($1.18 \times 10^{-38}$) with zero additional VRAM overhead.
+
+### Changed
+- **Unified Denominator Stability Constant ($\epsilon_{\text{eff}} = \epsilon = 10^{-8}$)**:
+  * Unified the effective denominator constant $\epsilon_{\text{eff}} = \epsilon = 10^{-8}$ across all precision modes and hardware backends (replacing the legacy `FP16` floor clamp $\max(\epsilon, 10^{-4})$ introduced in `1.1.5`), preventing update velocity attenuation and late-stage convergence loss resets when $\sqrt{v_t} \ll 10^{-4}$.
+
+### Removed
+- **Centered Defect Compensation (`centered_defect`, $\mu_d$)**:
+  * Removed the `centered_defect` hyperparameter and the global defect mean reduction $\mu_d = \frac{1}{N}\sum_{i=1}^N (g_{t,i} - m_{t-1,i})$ across both Dual-Buffer Convex Lookahead Momentum (`buffer=2` / CLM) and Single-Buffer Triangle-Defect Convex Lookahead Momentum (`buffer=1` / TD-CLM).
+  * Simplified the instantaneous gradient-momentum residual to purely local coordinate-wise differences:
+
+    $\displaystyle d_t^{(\text{B1})} = g_t - m_{t-1}, \qquad d_t^{(\text{B2})} = g_t - m_t$
+
+  * Eliminated the pre-step global reduction pass (`_lumina_diff_sum_kernel` in Triton JIT/AOT, `diff_sum.comp` in Vulkan SPIR-V, `reduce_diff_sum_simd` in x86\_64 AVX-512/AVX2, and slot 5 of the 6-element `ScratchBuffer`), removing cross-lane/cross-rank reduction latency while retaining the 6-element physical `ScratchBuffer` layout (`[mask_sum, u_sq_sum, p_sq_sum, cautious_scale, bound_scale, _reserved]`, 5 active slots + 1 reserved slot for ABI alignment).
+
+---
+
+## [1.5.3] - 2026-10-20
+
+No versions were published between 1.3.3 and 1.5.3. This release notes all changes made since version 1.3.3.
+
+### Added
+- **Single-Buffer Triangle-Defect Convex Lookahead Momentum (TD-CLM, `buffer=1`)**:
+  * Replaced the legacy 3-pass RMS single-buffer update with a single-pass formulation:
+
+    $\displaystyle \tilde{m} _ t = w_1 m _ {t-1} + w_2 g_t, \quad w_1 = \beta_1^2, \quad w_2 = 1 - \beta_1^2$
+
+    $\displaystyle \mathcal{E} _ t = w_1 |m _ {t-1}| + w_2 |g_t| + \lambda |d_t^{(\text{B1})}| + \epsilon _ {\text{eff}}$
+
+    $\displaystyle u_t^{(\text{B1})} = \frac{\tilde{m} _ t}{\mathcal{E} _ t} \in (-1, 1)$
+
+  * In 1.3.3, single-buffer mode required three sequential GPU kernel launches (`_lumina_v1_pass1_kernel`, `_lumina_v1_pass2_kernel`, `_lumina_v1_pass3_update_kernel`) and global atomic reductions across thread blocks to compute tensor-wide root-mean-square ($\text{rms} = \sqrt{\frac{1}{N}\sum \tilde{m} _ i^2 + \epsilon}$).
+  * TD-CLM replaces the tensor-wide reduction with coordinate-wise envelope bounding via the Triangle Inequality Envelope $\mathcal{E} _ t$. This reduces single-buffer execution to a single GPU pass and eliminates all $\tanh$ evaluations in the momentum path.
+  * Calibrated the defect penalty weight $\lambda$ to the symmetric spike-reversal equilibrium:
+
+    $\displaystyle \lambda^* = \frac{2(1 - \beta_1^2)^2}{4\beta_1^2 - 3}$
+
+    For the default $\beta_1 = 0.9$, $\lambda^* \approx 0.3008$, setting the default parameter to `lam=0.3`.
+
+- **Padé [13/6] Rational Approximation (`tanh_pade_f32`)**:
+  * Implemented a rational Padé [13/6] polynomial for hyperbolic tangent calculations:
+
+    $\displaystyle \tanh _ {\text{Pade}}(x) = \frac{x(1 + c_1 x^2 + c_2 x^4 + c_3 x^6 + c_4 x^8 + c_5 x^{10} + c_6 x^{12})}{1 + d_1 x^2 + d_2 x^4 + d_3 x^6}$
+
+  * Evaluated over the clamped domain $x \in [-7.9988117, 7.9988117]$ with asymptotic saturation to $\pm 1.0$.
+  * In 1.3.3, `_triton_tanh_fast` used hardware-dependent SFU instructions (`tl.math.tanh`) or analytical sigmoid fallbacks (`2 / (1 + exp(-2x)) - 1`). Different GPU architectures (e.g. Turing, Ampere, Hopper) evaluate SFU transcendental functions with slightly different mantissa rounding, leading to bitwise divergence during distributed training.
+  * Padé [13/6] provides uniform precision with $\lVert \tanh _ {\text{Pade}}(x) - \tanh(x) \rVert _ \infty < 1.5 \times 10^{-6}$ across CPU Scalar, AVX2, AVX-512, Vulkan SPIR-V, and Triton GPU kernels.
+
+- **Centered Innovation Defect (`centered_defect`)**:
+  * Added batch-mean centering for gradient innovation tracking (`centered_defect=False` by default):
+
+    $\displaystyle \mu_d = \frac{1}{N}\sum _ {i=1}^N (g _ {t,i} - m _ {t-1,i})$
+
+    $\displaystyle d_t^{(\text{B1})} = (g_t - m _ {t-1}) - \mu_d$
+
+    $\displaystyle d_t^{(\text{B2})} = (g_t - m_t) - \beta_1 \mu_d$
+
+  * In 1.3.3, uncentered innovation $g_t - m_t$ was used directly. On layers where gradients have a non-zero mean (such as normalization scales, biases, and token embeddings), the raw difference caused systematic drift in the second-moment accumulator. Centering eliminates this mean shift.
+
+- **Subnormal-Safe IEEE-754 FP16 Stochastic Rounding**:
+  * Added `f32_to_f16_bits_rne` to handle IEEE-754 subnormal numbers ($|x| < 2^{-14}$) during FP16 stochastic rounding.
+  * In 1.3.3, stochastic rounding added 13-bit random noise and masked with `& -8192`. When a floating-point value entered the subnormal range, bitwise addition directly corrupted the exponent field (bit 10), causing values to flip between normal and subnormal representations or trigger hardware flush-to-zero traps.
+  * The updated routine checks for zero exponent, shifts the significand into the subnormal format, and applies round-to-nearest-even tie-breaking before bit truncation.
+
+- **Multi-Backend Execution Engine**:
+  * Replaced the Python-only runtime with a multi-backend execution layer:
+    * **Rust SIMD (`luminav-simd`)**: AVX-512 (512-bit registers), AVX2 with FMA, portable scalar fallback, and Rayon parallel chunk scheduling (`ELEM_CHUNK_SIZE = 2,097,152`).
+    * **Vulkan UMA (`luminav-vulkan`)**: SPIR-V compute pipeline for cross-vendor GPUs (AMD, Intel, Apple Silicon via MoltenVK, and NVIDIA) with zero-copy host-visible memory buffers.
+    * **Triton AOT (`luminav-triton-aot`)**: C-ABI wrapper dispatching ahead-of-time precompiled kernels for 84 GPU architectures (Blackwell SM100/SM120, Hopper SM90, Ada SM89, Ampere SM80/SM86, Turing SM75, and AMD ROCm gfx90a/gfx942/gfx1100).
+    * **Standalone Runtime (`luminamoon.optimizer.standalone`)**: Pure NumPy and C-ABI execution that runs without requiring PyTorch.
+
+### Changed
+- **Parameter `alpha_ss`**: Removed. In 1.3.3, `alpha_ss=0.5` was used for soft-sign scaling $z / (1 + \alpha _ {\text{ss}} |z|)$ in the legacy single-buffer path. In 1.5.3, TD-CLM envelope normalization bounds updates directly to $(-1, 1)$, making soft-sign obsolete.
+- **Parameter `lam`**: Added with default `0.3`. Governs the defect penalty weight in TD-CLM.
+- **Parameter `centered_defect`**: Added with default `False`. Enables batch-mean centering of the innovation defect.
+- **Parameter `bound_type`**: Default changed from `"radial"` (1.3.3) to `"coordinate"` (1.5.3). Coordinate-wise hyperbolic bounding $(B_i \tanh(\Delta \theta_i / B_i))$ evaluates locally on each parameter without computing a tensor-wide norm reduction.
+- **Parameter `bound_ratio`**: Default changed from `0.03` to `0.05` to provide better update headroom on large model architectures.
+- **Parameter `ema_decay`**: Default changed from static `0.8` to `"auto"`. Resolves to `0.80` for Dual-Buffer mode (`buffer=2`) and `0.95` for Single-Buffer mode (`buffer=1`).
+- **Scratch Buffer**: Expanded from 5 floats `[mask_sum, u_sq_sum, p_sq_sum, cautious_scale, bound_scale]` to 6 floats, adding `diff_sum` at index 5 to accumulate batch-mean defect statistics.
+- **Cautious Scaling Floor**: Enforced `cautious_clamp_min=0.5` across all backends to cap maximum cautious energy scaling at $2.0\times$ ($2^1$).
+
+### Fixed
+- **Hardware Mantissa Divergence**: Fixed bitwise output differences across GPU families by replacing hardware SFU `tl.math.tanh` with Padé [13/6] rational approximation.
+- **FP16 Subnormal Bit Corruption**: Fixed exponent corruption in subnormal FP16 stochastic rounding by separating exponent extraction from significand shift logic.
+- **Single-Buffer Kernel Launch Latency**: Reduced single-buffer kernel dispatches from 3 passes to 1 pass, eliminating host-side synchronization bubbles and intermediate L2 atomic reduction overhead.
+- **Non-Zero Mean Gradient Drift**: Resolved directional accumulator drift on bias and normalization layers via `centered_defect`.
+
+### Parameter Comparison (1.3.3 vs 1.5.3)
+
+| Parameter | Version 1.3.3 | Version 1.5.3 | Technical Notes |
+| :--- | :--- | :--- | :--- |
+| `params` | `params` | `params` | Iterable of parameters or parameter groups. |
+| `lr` | `8e-4` | `8e-4` | Base learning rate. |
+| `betas` | `(0.9, 0.999)` | `(0.9, 0.999)` | First and second moment exponential decay rates. |
+| `eps` | `1e-8` | `1e-8` | Numerical stability floor. |
+| `weight_decay` | `8e-2` | `8e-2` | Decoupled weight decay rate. |
+| `tau` | `0.8` | `0.8` | Noise dispersion scale factor. |
+| `alpha_ss` | `0.5` | *(removed)* | Removed; superseded by TD-CLM envelope normalization. |
+| `lam` | *(none)* | `0.3` | Defect penalty weight in TD-CLM (calibrated to $\lambda^* \approx 0.3008$). |
+| `cautious` | `True` | `True` | Directional verification gate ($u_i g_i > 0$). |
+| `cautious_clamp_min` | `0.5` | `0.5` | Maximum cautious scale cap ($2.0\times$). |
+| `buffer` | `2` | `2` | `1` = Single-Buffer (TD-CLM), `2` = Dual-Buffer (CLM). |
+| `centered_defect` | *(none)* | `False` | Enables subtraction of batch-mean defect $\mu_d$. |
+| `stochastic_rounding` | `True` | `True` | Bitwise stochastic rounding on FP16/BF16 weights. |
+| `bound` | `True` | `True` | Enables update trust-region bounding. |
+| `bound_type` | `"radial"` | `"coordinate"` | Default changed to coordinate-wise bounding (zero reduction overhead). |
+| `bound_ratio` | `0.03` | `0.05` | Maximum step magnitude relative to parameter magnitude. |
+| `master_weights` | `False` | `"none"` | `"none"`, `"semi"`, or `"full"`. |
+| `execution` | `"auto"` | `"auto"` | Dynamic routing: SIMD, Vulkan, Triton, or Foreach. |
+| `fused_single_pass` | `False` | `True` | Enables single-pass streaming with EMA scaling. |
+| `ema_decay` | `0.8` | `"auto"` | Automatically sets `0.80` for dual-buffer, `0.95` for single-buffer. |
+| `fsdp_sync` | `False` | `False` | Cross-rank collective all-reduce synchronization. |
+| `seed` | `1337` | `1337` | PRNG seed for deterministic stochastic rounding. |
+
+---
+
+## [1.3.3] - 2026-10-06
+
+### Fixed
+* **Triton AST Compiler Early-Return Inlining Error:**
+  * Resolved fatal MLIR/CFG code generation failure (`at 35:4: _store_state_buffer(...) ^`) triggered by mid-branch `return` statements in `@triton.jit` helper functions (`_store_param` and `_store_state_buffer`).
+  * Refactored helper control flow into structured, single-exit `if ... elif ... else` branches, ensuring compliant JIT function inlining across all kernel entry points.
+* **Bitcast Width Mismatch (Int64 Promotion in Stochastic Rounding):**
+  * Fixed implicit 64-bit integer promotion caused by Python integer literals (`0x9E3779B9`, `~0x1FFF`, `~0xFFFF`) inside Triton bitwise operations.
+  * Enforced strict 32-bit signed arithmetic via two's complement constants (`-1640531527`, `-2048173461`, `-1028477387`) with explicit `.to(tl.int32)` coercion, and standardized bitwise masks (`-65536` for BF16, `-8192` for FP16), ensuring type-safe 32-bit bitcasting (`tl.int32` to `tl.float32`).
+* **JIT Kernel Call Keyword Arguments:**
+  * Replaced internal keyword argument invocations (`use_sr=False`) with standard positional boolean parameters inside Triton JIT kernels to eliminate compiler argument binding warnings across Triton runtime versions.
+
+### Improved
+* **Throttled Fallback Telemetry & Console De-Noising:**
+  * Added `self._warned_triton_failure` state tracking to prevent multi-line compiler AST dumps from flooding the console across every parameter tensor in a parameter group.
+  * Formatted fallback notifications into a single concise one-line `WARNING` log on initial encounter, routing full tracebacks to `DEBUG` level.
+
+### Verified
+* **End-to-End JIT Execution on NVIDIA Architectures (Tesla T4):**
+  * Validated JIT execution across both Dual-Pass (`fused_single_pass=False`) and Fused Single-Pass (`fused_single_pass=True`) modes in native FP16.
+  * Confirmed non-zero weight update dynamics ($|\Delta W| > 0$) without gradient underflow or weight freezing in pure master-free training for Version 1.3.3.
+
+---
+
+## [1.3.2] - 2026-10-06
+
+### Added
+- **Distributed FSDP Collective Synchronization (`fsdp_sync`)**:
+  - Introduced the `fsdp_sync: bool = False` configuration flag on optimizer initialization, backed by automated runtime guarding via `torch.distributed.is_initialized()`.
+  - Integrated cross-GPU collective communications (`torch.distributed.all_reduce(..., op=torch.distributed.ReduceOp.SUM)`) for global reduction metrics (`mask_sum`, `u_sq_sum`, and `p_sq_sum`) across Triton kernels, single-step execution, and vectorized foreach loops.
+  - Maintains bit-exact scalar consistency for `m_bar`, `cautious_scale`, and `bound_scale` across distributed parameter shards during large-scale pretraining under PyTorch FSDP (Fully Sharded Data Parallel) and DeepSpeed ZeRO-3.
+- **Dedicated Low-Precision Stochastic Quantization Routine (`_sr_quantize`)**:
+  - Factored out bitwise stochastic rounding into a dedicated, modular helper function: `_sr_quantize(target, val_f32, use_sr, generator)`.
+  - Enabled native stochastic rounding on half-precision momentum states (`exp_avg` in BF16/FP16) across all PyTorch CPU/CUDA fallback paths (`_single_step_v2`, `_foreach_step_v2`, `_single_step_v1`, and `_foreach_step_v1`), systematically eliminating gradient stagnation induced by fractional bit truncation (LSB underflow).
+- **Persistent Contiguous Buffer Caching**:
+  - Introduced persistent state cache tensors within `self.state[p]` (`p_contig`, `exp_avg_contig`, `exp_avg_sq_contig`, and `mp_contig`) to stage strided or sliced parameters before kernel launch.
+  - Eliminates dynamic memory allocations (`.contiguous()`) during iterative training loops, preserving PyTorch VRAM caching allocator stability and preventing CUDA Graph capture invalidation.
+- **Zero-Copy Direct Tensor Aliasing for FP32 Master Weights**:
+  - When parameter tensors are natively `torch.float32`, `state["master_param"]` now directly aliases the parameter tensor pointer (`state["master_param"] = p`), avoiding additional master weight VRAM overhead for FP32 layers (0 bytes auxiliary memory footprint).
+- **Checkpoint Sanitizer via State Dict Filter (`state_dict`)**:
+  - Overrode `state_dict()` to automatically purge temporary contiguous cache buffers (`p_contig`, `exp_avg_contig`, `exp_avg_sq_contig`, `mp_contig`), ensuring on-disk checkpoint payloads remain clean, compact, and fully backward-compatible.
+- **Defensive Device Pipeline Flush on Kernel Fault (`torch.cuda.synchronize`)**:
+  - Embedded explicit hardware device synchronization (`torch.cuda.synchronize(p.device)`) within Triton exception recovery handlers prior to PyTorch fallback execution, preventing asynchronous CUDA error cascades into subsequent tensor operations.
+
+### Changed
+- **High-Entropy Bit-Avalanche PRNG Hash Function (`_prng`)**:
+  - Replaced the standard Linear Congruential Generator (LCG) with a MurmurHash3 / SplitMix32 integer bit-avalanche hashing routine:
+    ```python
+    z = idx + seed + 0x9E3779B9
+    z = (z ^ (z >> 16)) * 0x85EBCA6B
+    z = (z ^ (z >> 13)) * 0xC2B2AE35
+    z = z ^ (z >> 16)
+    ```
+  - Delivers uniform pseudo-random noise distributions with zero low-order bit periodicity, significantly refining stochastic rounding fidelity on BF16/FP16 parameters.
+- **Memory-Contiguous Scratch Buffer Realignment for v1 Triton Kernels**:
+  - Re-indexed reduction buffer offsets in Triton v1 kernels:
+    - Pass 1 metrics (`[rms_sum, p_sq_sum]`) mapped to contiguous indices `[0, 1]`.
+    - Pass 2 metrics (`[mask_sum, u_sq_sum]`) mapped to contiguous indices `[2, 3]`.
+  - Enables vectorized contiguous slice all-reduces (`scratch_param[:2]` and `scratch_param[2:4]`) under FSDP without requiring non-contiguous memory gather/scatter overhead.
+- **Multi-GPU Device Index Resolution in RNG Dispatcher (`_get_generator`)**:
+  - Fixed device index resolution when `device` is passed as a generic `torch.device("cuda")` without an explicit index by defaulting to `torch.cuda.current_device()`, avoiding cross-GPU generator key collisions during multi-GPU distributed runs.
+- **Conditional Atomic Reduction Pruning in v2 Pass 1 Kernel**:
+  - Guarded `mask_sum` atomic accumulation in `_lumina_v2_pass1_kernel` with `if cautious:`, bypassing CTA reductions and L2 atomic additions when `cautious=False`.
+- **Expanded Master Weight Precision Configuration Aliases**:
+  - Added the `"half"` alias for `"semi"` mode alongside `"fp32"` and `"masterweight"` aliases for `"full"` mode.
+- **Metadata & Documentation Header Update**:
+  - Bumped specification version to `1.3.2`.
+
+### Fixed
+- **Singular Zero-Division & Step-Zero Bias Correction (`safe_base_step`)**:
+  - Introduced `safe_base_step = max(base_step, 1)` and `bc2_base = max(1.0 - (beta2 ** safe_base_step), 1e-15)` across Triton, single-step, and foreach loops.
+  - Fully resolves potential `ZeroDivisionError` and NaN blowups in `c1` / `c2` computations when steps evaluate at `step == 0`.
+- **Numerical Overflow Clamp in Sigmoid-Based Tanh Fallback**:
+  - Applied register clamp bounds `x_clamped = tl.maximum(tl.minimum(x, 20.0), -20.0)` in analytical fallback `_triton_tanh_fast`, preventing `tl.exp(-2.0 * x)` from overflowing to `inf` or `nan` on devices lacking hardware SFU tanh instructions.
+- **Missing Argument Forwarding in PyTorch Single-Step v1**:
+  - Corrected `_single_step_v1` method signature and call-sites to accept and propagate `scratches`, eliminating runtime `TypeError` during fallback invocation.
+- **Multi-Rank Synchronization Skew in Sharded Parameter Bounding**:
+  - Aligned tensor norms and radial bounding factors across sharded ranks via synchronized global scalar reductions, preventing parameter update drift across distributed ranks.
+
+### Performance & Engine Refactoring
+- **Zero Dynamic VRAM Allocation on Strided / Sliced Tensors**:
+  - Persistent pre-allocated contiguous staging buffers avoid PyTorch caching allocator churn and eliminate CUDA Graph memory address invalidations.
+  - Preserves deterministic memory footprints across long pre-training runs.
+- **Optimized Distributed Collective Bandwidth**:
+  - Grouping scalar reductions into contiguous vector slices compresses distributed synchronization overhead into a single collective kernel launch per pass.
+
+---
+
+## [1.3.0] - 2026-09-23
+
+### Added
+- **Fused Single-Pass Kernel with Post-Step EMA (`fused_single_pass`, `ema_decay`)**:
+  - Introduced `_lumina_v2_fused_single_pass_kernel` coupled with an asynchronous CTA micro-kernel `_lumina_v2_post_step_ema_kernel[(1,)]`.
+  - When `fused_single_pass=True`, avoids two-pass global reductions by leveraging running Exponential Moving Average (EMA) estimates of `cautious_scale` ($1 / \bar{M}$) and `bound_scale` ($\tanh(r) / r$).
+  - Unifies gradient filtering, momentum accumulation, variance dispersion, cautious masking, radial bounding, decoupled weight decay, stochastic rounding, and block partial reductions into a **single unified VRAM-resident pass**, reducing global GPU memory bandwidth traffic per optimizer step.
+- **Hybrid Master Weight Precision Architecture (`master_weights`)**:
+  - Added multi-tier master weight flexibility via `master_weights: Union[bool, str] = False`:
+    - `"none"` / `False` *(Default)*: Pure master-free low-precision training leveraging on-chip bitwise stochastic rounding directly on native half-precision weights.
+    - `"semi"` / `"hybrid"` / `"half"`: 32-bit FP32 master parameters (`state["master_param"]`) coupled with 16-bit low-memory optimizer states (`exp_avg`, `exp_avg_sq` in BF16/FP16), saving ~50% optimizer state VRAM compared to traditional FP32 setups while maintaining full FP32 weight accumulation accuracy.
+    - `"full"` / `"fp32"` / `"masterweight"`: Traditional full FP32 master parameters with full FP32 optimizer states for standard baseline comparisons.
+- **In-Kernel In-Flight NaN / Inf Gradient Sanitization**:
+  - Embedded register-level defensive sanitization across all Triton GPU kernels: `is_finite_g = (g == g) & (g < 1e38) & (g > -1e38); g = tl.where(is_finite_g, g, 0.0)`.
+  - Replaces non-finite gradients and subnormal corruptions with zero in registers before they can contaminate momentum (`exp_avg`) or variance (`exp_avg_sq`) state buffers.
+- **Low-Precision State Buffer Serialization (`_store_state_buffer`)**:
+  - Added a dedicated Triton state serialization routine supporting FP32, BF16, and FP16 state representations with optional bitwise stochastic rounding applied directly to momentum buffers.
+- **Multi-Device & Cross-Accelerator Hardware Context Isolation (`_get_device_context`)**:
+  - Unified device stream scoping supporting NVIDIA CUDA (`torch.cuda`), Intel XPU (`torch.xpu`), and modern PyTorch 2.x unified accelerator interfaces (`torch.accelerator.device`), eliminating multi-GPU stream hijacking and cross-device race conditions.
+- **Strict Determinism Auto-Routing**:
+  - Automatically queries `torch.are_deterministic_algorithms_enabled()`. When enabled under `execution="auto"`, dynamically routes execution to C++ vectorized `foreach` to preserve bitwise cross-run reproducibility, circumventing non-deterministic floating-point hardware atomic reordering.
+- **Granular Fault Recovery with State Guarding (`completed_indices`)**:
+  - Implemented per-parameter completion tracking in `_dispatch_triton`. If a kernel encounters an unexpected runtime fault, completed parameters are preserved while uncompleted ones fall back cleanly to `foreach`.
+  - Added `_single_step_v2_apply_param` and `_single_step_v1_apply_param` to safely finalize parameter updates if Pass 1 had already committed state transitions, strictly preventing double-update state corruption.
+
+### Changed
+- **Per-Parameter Persistent Scratch Tensors (Elimination of Global Slicing)**:
+  - Replaced the global dynamic slice allocator (`self._scratch_tensors[dev][:count]`) with isolated per-parameter persistent state allocations (`state["scratch"] = torch.zeros((5,), device=p.device, dtype=torch.float32)`).
+  - Memory pointers for `[mask_sum, u_sq_sum, p_sq_sum, cautious_scale, bound_scale]` remain completely invariant throughout the entire training lifecycle, preserving pointer stability for **CUDA Graph Capture** (`torch.accelerator.Graph`) and TorchInductor AOT compilation without memory aliasing.
+- **Analytically Exact Nesterov Bias Correction ($\beta_1^{\text{step} + 1}$)**:
+  - Corrected first-moment bias correction from the classical Adam formulation ($1.0 - \beta_1^{\text{step}}$) to the exact analytical expectation of the Nesterov blended momentum: `bc1_nes = 1.0 - beta1^(step + 1)`. Removes residual momentum bias during early warmup iterations.
+- **Adaptive CTA Tile Sizing and Dynamic Thread Warp Scaling**:
+  - Introduced hardware-aware CTA tile scaling based on parameter volume: parameters with $\ge 2,097,152$ elements (2M+) dynamically configure `BLOCK_SIZE = 2048` and `num_warps = 8` (versus `BLOCK_SIZE = 1024`, `num_warps = 4` for smaller tensors).
+  - Cuts total thread-block grid counts and L2 cache atomic addition pressure by 50% on large MLP, Attention Projection, and Vocabulary layers.
+- **Hardware-Native Tanh Instruction Dispatch (`tl.math.tanh` / `tl.tanh`)**:
+  - Upgraded `_triton_tanh_fast` to dynamically inspect compiler capabilities, binding directly to native PTX Special Function Unit (SFU) instructions (`tanh.approx.f32`) on modern Triton compilers while retaining analytical sigmoid fallbacks for legacy environments.
+- **PRNG Decorrelation between Momentum and Parameter Updates**:
+  - Separated RNG seed streams using bitwise alternating mask shifts: `seed_p = (seed + 0x55555555) & 0x7FFFFFFF`, mathematically eliminating cross-coupling noise correlation between state buffer rounding and parameter update rounding.
+- **Bumped specification version metadata to 1.3.0** across `luminav.py`.
+
+### Fixed
+- **Floating-Point Cancellation in Radial Bounding ($r \to 0$)**:
+  - Fixed precision jitter and roundoff degradation near zero by adding an asymptotic Taylor-series guard across Triton and PyTorch bounding engines: `torch.where(r < 1e-4, 1.0, tanh(r) / r)`.
+- **Artificial Denominator Dilation in Dispersion Scale ($\sigma$)**:
+  - Replaced additive epsilon inflation (`sigma + 1e-6`) with a non-distorting floor clamp (`tl.maximum(sigma, 1e-12)`), preventing artificial dampening of subtle gradient signals.
+- **Singular Zero-Division in Parameter Norm Clamping**:
+  - Replaced `norm_p + 1e-6` with `torch.clamp_min(norm_p, 1.0)` / `tl.maximum(norm_p, 1.0)`, maintaining numerical stability even when parameters are initialized near zero.
+- **Transient Memory Pool Spikes during PyTorch Fallback Stochastic Rounding**:
+  - Refactored `_sr_update` to stream stochastic noise generation in bounded chunks of 1,048,576 elements (~4 MB chunk size), eliminating multi-hundred megabyte temporary int32 tensor spikes during fallback execution on large parameter matrices.
+- **Non-Contiguous Buffer Safety & Strided Memory Copy**:
+  - Added strict contiguous layout verification for parameters, states, and master weights with in-place synchronization (`p.copy_(p_contig)`), guarded by a one-time developer warning.
+
+### Performance & Engine Refactoring
+- **~50% Global Memory Bandwidth Reduction via Fused Single-Pass Mode**:
+  - By unifying parameter loading, gradient evaluation, state transformation, and bounding into one GPU pass under `fused_single_pass=True`, VRAM read/write cycles are halved for cautious and radially bounded steps.
+- **Sub-Microsecond L2 Hardware Atomic Reduction Architecture**:
+  - Formally documented the hybrid Intra-Block Tree Reduction + Inter-Block Hardware Atomic pattern. With `BLOCK_SIZE=2048`, a 10M-parameter tensor issues only 4,883 atomic additions directly into GPU L2 cache crossbars, executing in < 0.5 µs and strictly outperforming multi-tier cascaded reduction micro-kernels by eliminating 3–5 µs driver dispatch bubbles.
+- **Static Buffer Address Invariance for CUDA Graphs & TorchInductor AOT**:
+  - Persistent 5-slot state scratch buffers eliminate all dynamic slicing and memory reallocations during optimization loops, preserving pointer address invariance for CUDA Graph replay.
+
+---
+
+## [1.2.1] - 2026-09-17
+
+### Added
+- **In-Kernel GPU Parameter Norm Reduction (`p_sq_sum_ptr`)**: Fused parameter Euclidean norm squared $\sum(p_i^2)$ calculation directly into Pass 1 Triton reduction kernels (`_lumina_v2_pass1_kernel` and `_lumina_v1_pass1_kernel`). Pass 2 and Pass 3 now load and resolve `norm_p` (`tl.sqrt(tl.maximum(p_sq_sum, 0.0))`) entirely inside GPU SRAM/registers without host intervention.
+- **Stateless CPU Integer Multiplicative PRNG Hashing**: Replaced dynamic GPU RNG tensor allocation with a pure CPU integer hash using Knuth's 32-bit golden ratio constant: `(step * 0x9E3779B9 + i * 10007) & 0x7FFFFFFF`. Generates uncorrelated 31-bit integer seeds per layer with zero GPU kernel launch overhead and zero PCIe host-device latency.
+- **Isolated Generator Cache for Fallback Engine (`self._generators`)**: Added an internal per-device `torch.Generator` cache, ensuring stochastic rounding in `_sr_update` operates in complete isolation from PyTorch's default CUDA generator state.
+
+### Changed
+- **Consolidated Scratch Buffer Zeroing**: Removed redundant per-parameter slice zeroing inside parameter loops. Buffer zeroing is now batched into a single consolidated `scratch.zero_()` call per step inside `_get_scratch_buffer`.
+- **Scratch Buffer Stride Expansion**:
+  - `buffer=2`: Expanded stride from 2×N to 3×N elements per parameter to host `[mask_sum, u_sq_sum, p_sq_sum]`.
+  - `buffer=1`: Expanded stride from 3×N to 4×N elements per parameter to host `[rms_sum, mask_sum, u_sq_sum, p_sq_sum]`.
+- **Bumped specification version metadata to 1.2.1**.
+
+### Fixed
+- **Host-Device Synchronization & Pipeline Stalls (`.item()` Bottleneck)**: Removed `norm_p = float(p_contig.float().norm().item())` and `seed = torch.randint(...).item()` from Triton step pipelines. Eliminated 200–400 synchronous PCIe roundtrips and GPU pipeline stalls per step on deep architectures (e.g., 28-layer Transformers), restoring full asynchronous GPU compute utilization.
+- **CUDA Graphs & `torch.compile` Graph Breaks**: Fixed fatal graph capture crashes and TorchDynamo graph breaks caused by blocking host-device synchronization during radial bounding and seed generation. The optimizer step is now compatible with CUDA Graphs (`torch.accelerator.Graph`) and TorchInductor compilation.
+- **Transient VRAM Allocator Thrashing**: Eliminated repeated temporary FP32 parameter cloning (`p_contig.float()`) during norm evaluation, preventing memory pool fragmentation in tight VRAM regimes.
+- **Global RNG Pollution & Determinism Compliance**: Fixed global RNG state mutation in fallback stochastic rounding. Solved reproducibility drift in upstream Dropout, DataLoader shuffling, and RLHF reward modeling, ensuring strict adherence to PyTorch 2.10+ `torch.use_deterministic_algorithms(True)`.
+
+### Performance & Engine Refactoring
+- **Batch Kernel Launch Overhead Elimination**: Removed up to 400–800 individual `cudaMemsetAsync` micro-calls per step previously triggered by per-slice `.zero_()` invocations on 1-element scratch tensors.
+- **Single-Pass Bypass for Coordinate Bounding**: Streamlined `_triton_step_v2` to immediately bypass two-pass reductions and invoke `_lumina_v2_single_pass_kernel` when `cautious=False` and coordinate bounding (`bound_mode == 2`) is selected.
+
+---
+
+## [1.2.0] - 2026-09-17
+
+### Added
+- **Automatic Dynamic FP16 Cliff Governor:** Built-in asymptotic boundary governor (`_triton_tanh_fast` in GPU register and PyTorch fallback) that automatically activates when `p.dtype == torch.float16`. Prevents unclipped steps from pushing parameters, activations, and residual streams into the IEEE-754 FP16 overflow cliff (> 65,504). Remains completely dormant (zero overhead) on `bfloat16` (TPU / Ampere+) and `float32`.
+- **Smooth Asymptotic Parameter Bounding (`bound`, `bound_type`, `bound_ratio`):** Added non-intrusive step bounding with two mathematically continuous formulations:
+  - `"radial"`: Direction-Preserving Radial Squashing (`update * tanh(r) / r`) preserving gradient direction (Cosine Similarity = 1.000).
+  - `"coordinate"`: Dual-Hyperbolic Coordinate Squashing for fine-grained per-element damping.
+  - Configurable via constructor: `bound: bool = True`, `bound_type: str = "radial"`, `bound_ratio: float = 0.03`.
+
+### Changed
+- **Increased Default `cautious_clamp_min` from `0.2` to `0.5`:** Enforces an exact power-of-two maximum energy scaling factor (2.00x, $2^1$). Eliminates floating-point mantissa roundoff distortion in IEEE-754 hardware.
+- Bumped specification version metadata to `1.2.0`.
+
+### Fixed
+* **FP16 Step Divergence & NaN Overflow at High Learning Rates:** Solved step divergence and subsequent NaN crashes when training deep 28-layer Transformer architectures (e.g., Qwen3-0.6B) in pure FP16 at nominal learning rates (`lr = 8e-4`).
+* **Sparse Embedding & LM-Head Amplification:** Mitigated 5x over-amplification on sparse vocabulary gradients where active token counts are much smaller than total tensor parameters ($N \gg N_{\text{active}}$).
+* **Verified stable 1,000-step pretraining from scratch on Qwen3-0.6B** (seq length 512, pure FP16, `lr = 8e-4`) on Nvidia Tesla T4 with zero gradient clipping and zero LR scheduler (loss dropped from 12.09 down to ~5.7 with zero NaNs).
+* **Weight Decay Numerical Parity on Fallback Engine:** Resolved a silent precision-stagnation issue in `_single_step` and `_foreach_step`. Previously, decoupled weight decay (`p.mul_(1.0 - lr * weight_decay)`) was executed directly on native FP16/BF16 tensors prior to `_sr_update`. Due to IEEE-754 Round-to-Nearest (RTN) limits, `(1 - lr * weight_decay) ≈ 0.999936` rounded back to `1.0`, causing weight decay to freeze on half-precision parameters. Weight decay is now unified into FP32 registers inside `_sr_update` alongside stochastic rounding, matching the numerical behavior with the Triton kernel.
+* **Fallback Foreach Indentation & Scope Alignment:** Corrected tensor iteration scope and indentation for `torch._foreach_add_` in non-SR fallback branches.
+* **In-Place GPU Clamping in `_apply_bound`:** Replaced dynamic GPU scalar allocations (`torch.tensor(1.5e-4, device=p.device)`) with native `torch.clamp_min`, eliminating transient CUDA malloc spikes per update step.
+
+### Performance & Engine Refactoring
+* **Unified Bound Execution Pipeline:** Added an immediate short-circuit guard (`if lr == 0.0: return update`) and streamlined the execution flow: intermediate updates are scaled in FP32 (`step_val = lr * u_float`), passed through radial/coordinate bounding, chained cleanly into the FP16 Cliff Governor, and returned in a single normalized step.
+
+---
+
+## [1.1.5] - 2026-09-15
+
+### Added
+- **Native Hardware Capability Detection (`_supports_native_bf16`):** Added automatic device capability detection via `torch.cuda.is_bf16_supported()` to identify Ampere SM80+ architectures and streamline native execution paths.
+- **Hugging Face Repository Reference:** Added official Hugging Face repository link in module docstrings.
+
+### Changed
+- **Dynamic Epsilon Clamping for FP16:** Dynamically floors effective epsilon to `max(eps, 1e-4)` when parameters or states are in `torch.float16`. Prevents bias-corrected scale factor `c2` from underflowing into IEEE-754 subnormal/zero limits on step 1.
+- Bumped specification version to `1.1.5`.
+
+### Fixed
+- **FP16 Step-1 `NaN` from Epsilon Underflow:** Fixed `0.0 / 0.0 = NaN` division occurring on initial training steps when parameters receive zero gradients (e.g., unselected vocabulary embeddings, padding tokens, or dropout paths).
+- **Second-Moment Variance Saturation (`+inf`):** Added safe upper-bound register clamping (`tl.minimum(v_new, 65000.0)` in Triton kernels and `.clamp_(0.0, 65000.0)` in PyTorch C++ foreach/single loops). Prevents unclipped initial pretraining gradient surges ($|g| > 256$) from overflowing the FP16 maximum dynamic range (65,504) into irreversible `+inf`.
+- **Zero-Sigma Division Guard:** Injected explicit `sigma + 1e-6` denominator guards across all Triton kernel passes and PyTorch update steps to ensure numerical safety during cold-start iterations.
+- Verified stable 100-step pretraining from scratch on `SmolLM-135M` using Wikimedia Wikipedia in pure FP16 on Nvidia Tesla T4 GPU with zero gradient clipping.
+
+---
+
+## [1.1.0] - 2026-09-11
+
+### Added
+- **Bitwise Stochastic Rounding (SR):** Integrated on-chip PRNG integer bit-manipulation hash in Triton kernels and vectorized PyTorch fallback for native FP16/BF16 training without master weights.
+- Unified Triton store routine introducing compile-time branching (`dtype_mode` / `use_sr: tl.constexpr`) with explicit safe casting for FP16 and BF16 pointers.
+- Vectorized fallback improvements utilizing C++ multi-tensor operations when stochastic rounding is disabled or tensors are FP32.
+- State buffer contiguity validation (`exp_avg_contig`, `exp_avg_sq_contig`) ensuring safe linear memory addressing before kernel dispatch.
+- Declarative configuration file (`config.json`) defining hyperparameters and optimizer presets.
+
+### Changed
+- Refactored PRNG integer arithmetic to strictly stay within signed 32-bit integer limits ($< 2^{31} - 1$), eliminating JIT compilation failures on newer Triton releases.
+- State buffers (`exp_avg`, `exp_avg_sq`) now initialize with strict contiguous memory format.
+
+### Fixed
+- Fixed `ValueError: Scalar out of range for type int32` and Triton LLVM compile errors during unsigned 32-bit bitmask operations.
+- Resolved kernel fallback hangs and compilation overhead on CUDA devices.
+
+---
+
+## [1.0.2] - 2026-09-08
+
+### Added
+- Official Apache-2.0 license notice and copyright headers.
+- Standardized package exports (`__all__`, `__version__`, `__author__`, `__license__`).
+
+---
+
+## [1.0.1] - 2026-08-31
+
+### Fixed
+- **Memory Synchronization Fix (`423d171`):** Fixed a critical bug where non-contiguous parameter tensors (e.g., transposed linear layers or sliced weights) were updated on temporary contiguous copies without reflecting changes back to model weights. Added `p.copy_(p_contig)` after Triton execution.
+
+---
+
+## [1.0.0] - 2026-08-30
+
+### Added
+- Initial release of the LuminaV optimizer.
+- **Zero Master-Weight Copies:** Direct parameter updates in native half-precision without allocating 4-byte FP32 master weights.
+- **Hyperbolic Tangent (`tanh`) Bounding Envelope:** Projects normalized momentum ratio into `(-1.0, 1.0)` to eliminate exploding gradient shocks.
+- **Directional Alignment Gate:** Cautious update masking that zeroes out momentum steps opposing mini-batch gradient directions.
+- **Central Innovation Variance:** Noise dispersion tracking $(g_t - m_t)^2$ decoupling mini-batch turbulence from directional descent.
+- Dual-buffer mode (`buffer=2`, LuminaV-2B) and single-buffer mode (`buffer=1`, LuminaV-1).
+- Custom fused OpenAI Triton kernels with automated fallback execution.

Cargo.lock

@@ -1,1218 +1,1218 @@
-# This file is automatically @generated by Cargo.
-# It is not intended for manual editing.
-version = 4
-
-[[package]]
-name = "ahash"
-version = "0.8.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75"
-dependencies = [
- "cfg-if",
- "const-random",
- "getrandom 0.3.4",
- "once_cell",
- "version_check",
- "zerocopy",
-]
-
-[[package]]
-name = "alloc-no-stdlib"
-version = "3.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c2fb6cfd47bf496ff64095c20eaba0c201404ee38714d4142fcfa1dc334fcc7a"
-
-[[package]]
-name = "alloc-stdlib"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b5c1865780388bfa186411ab5f247819487fc4864c6e9c3106611fa347586e1"
-dependencies = [
- "alloc-no-stdlib",
-]
-
-[[package]]
-name = "allocator-api2"
-version = "0.2.21"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
-
-[[package]]
-name = "android_system_properties"
-version = "0.1.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc"
-dependencies = [
- "libc",
-]
-
-[[package]]
-name = "arrow-array"
-version = "60.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0f3d22a41d3df78b71df0b685af219ae3a94188ab98d6fb29c958eb55c3247f4"
-dependencies = [
- "ahash",
- "arrow-buffer",
- "arrow-data",
- "arrow-schema",
- "chrono",
- "half",
- "hashbrown",
- "num-complex",
- "num-integer",
- "num-traits",
-]
-
-[[package]]
-name = "arrow-buffer"
-version = "60.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7bd46a530bc1a8e763102fc36c1dc0f51f3ffa150c78e7f5658ccb54b1458ade"
-dependencies = [
- "bytes",
- "half",
- "num-bigint",
- "num-traits",
-]
-
-[[package]]
-name = "arrow-cmp"
-version = "60.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1913e57a64fbf2c4502fa6d54b547d6a1ae59e3146cbc15321d47468c937abd1"
-dependencies = [
- "arrow-array",
- "arrow-buffer",
- "arrow-schema",
-]
-
-[[package]]
-name = "arrow-data"
-version = "60.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "415b7fbc4cf56b080bb5677a4ae74211b5c79e571ed37a1d547633ec07f3b43b"
-dependencies = [
- "arrow-buffer",
- "arrow-schema",
- "half",
- "num-integer",
- "num-traits",
-]
-
-[[package]]
-name = "arrow-ipc"
-version = "60.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "45f3e766bccf5a0cb57a5de52a4cba450dea5171547c2cb5939b486d08887cba"
-dependencies = [
- "arrow-array",
- "arrow-buffer",
- "arrow-data",
- "arrow-schema",
- "arrow-select",
- "flatbuffers",
-]
-
-[[package]]
-name = "arrow-schema"
-version = "60.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "de2c8e714b466404cdc1189e89f892426efea614d938e16c15518c340cd43ba3"
-
-[[package]]
-name = "arrow-select"
-version = "60.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a5af3f6656c5f2c6944f93d9a357a7f204c93d5f349385a44cf46c4a26fbcf30"
-dependencies = [
- "ahash",
- "arrow-array",
- "arrow-buffer",
- "arrow-cmp",
- "arrow-data",
- "arrow-schema",
- "num-traits",
-]
-
-[[package]]
-name = "ash"
-version = "0.38.0+1.3.281"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0bb44936d800fea8f016d7f2311c6a4f97aebd5dc86f09906139ec848cf3a46f"
-dependencies = [
- "libloading 0.8.9",
-]
-
-[[package]]
-name = "autocfg"
-version = "1.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
-
-[[package]]
-name = "base64"
-version = "0.23.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5"
-
-[[package]]
-name = "bitflags"
-version = "2.13.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06"
-
-[[package]]
-name = "brotli"
-version = "9.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8b851b75c23ca7873623d612fe49bd1989aeb03d08fb9432187eb253d3d4c6b"
-dependencies = [
- "alloc-no-stdlib",
- "alloc-stdlib",
- "brotli-decompressor",
-]
-
-[[package]]
-name = "brotli-decompressor"
-version = "6.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "941cd9bd4ddab83cb46fa5a2d428f1c857b24ac78cb876cf7beb710840934bd7"
-dependencies = [
- "alloc-no-stdlib",
- "alloc-stdlib",
-]
-
-[[package]]
-name = "bumpalo"
-version = "3.20.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
-
-[[package]]
-name = "bytes"
-version = "1.12.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
-
-[[package]]
-name = "cc"
-version = "1.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6651c9ed80effdc7db0ff72512157f901af5e3549e341e24b1dd4887d836d838"
-dependencies = [
- "find-msvc-tools",
- "jobserver",
- "libc",
- "shlex",
-]
-
-[[package]]
-name = "cfg-if"
-version = "1.0.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600"
-
-[[package]]
-name = "chrono"
-version = "0.4.45"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327"
-dependencies = [
- "iana-time-zone",
- "num-traits",
- "windows-link",
-]
-
-[[package]]
-name = "const-random"
-version = "0.1.18"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359"
-dependencies = [
- "const-random-macro",
-]
-
-[[package]]
-name = "const-random-macro"
-version = "0.1.16"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e"
-dependencies = [
- "getrandom 0.2.17",
- "once_cell",
- "tiny-keccak",
-]
-
-[[package]]
-name = "core-foundation-sys"
-version = "0.8.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
-
-[[package]]
-name = "crc32fast"
-version = "1.5.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78"
-dependencies = [
- "cfg-if",
-]
-
-[[package]]
-name = "crossbeam-deque"
-version = "0.8.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a"
-dependencies = [
- "crossbeam-epoch",
- "crossbeam-utils",
-]
-
-[[package]]
-name = "crossbeam-epoch"
-version = "0.9.21"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d"
-dependencies = [
- "crossbeam-utils",
-]
-
-[[package]]
-name = "crossbeam-utils"
-version = "0.8.23"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6"
-
-[[package]]
-name = "crunchy"
-version = "0.2.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
-
-[[package]]
-name = "either"
-version = "1.19.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be"
-
-[[package]]
-name = "equivalent"
-version = "1.0.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826"
-
-[[package]]
-name = "find-msvc-tools"
-version = "0.1.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484"
-
-[[package]]
-name = "flatbuffers"
-version = "25.12.19"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3"
-dependencies = [
- "bitflags",
- "rustc_version",
-]
-
-[[package]]
-name = "flate2"
-version = "1.1.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb"
-dependencies = [
- "zlib-rs",
-]
-
-[[package]]
-name = "foldhash"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
-
-[[package]]
-name = "futures-core"
-version = "0.3.34"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e"
-
-[[package]]
-name = "futures-task"
-version = "0.3.34"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd"
-
-[[package]]
-name = "futures-util"
-version = "0.3.34"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc"
-dependencies = [
- "futures-core",
- "futures-task",
- "pin-project-lite",
- "slab",
-]
-
-[[package]]
-name = "getrandom"
-version = "0.2.17"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0"
-dependencies = [
- "cfg-if",
- "libc",
- "wasi",
-]
-
-[[package]]
-name = "getrandom"
-version = "0.3.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd"
-dependencies = [
- "cfg-if",
- "libc",
- "r-efi 5.3.0",
- "wasip2",
-]
-
-[[package]]
-name = "getrandom"
-version = "0.4.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099"
-dependencies = [
- "cfg-if",
- "libc",
- "r-efi 6.0.0",
-]
-
-[[package]]
-name = "half"
-version = "2.7.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b"
-dependencies = [
- "cfg-if",
- "crunchy",
- "num-traits",
- "zerocopy",
-]
-
-[[package]]
-name = "hashbrown"
-version = "0.17.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
-dependencies = [
- "allocator-api2",
- "equivalent",
- "foldhash",
-]
-
-[[package]]
-name = "heck"
-version = "0.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
-
-[[package]]
-name = "iana-time-zone"
-version = "0.1.65"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470"
-dependencies = [
- "android_system_properties",
- "core-foundation-sys",
- "iana-time-zone-haiku",
- "js-sys",
- "log",
- "wasm-bindgen",
- "windows-core",
-]
-
-[[package]]
-name = "iana-time-zone-haiku"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
-dependencies = [
- "cc",
-]
-
-[[package]]
-name = "itoa"
-version = "1.0.18"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
-
-[[package]]
-name = "jobserver"
-version = "0.1.35"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3"
-dependencies = [
- "getrandom 0.4.3",
- "libc",
-]
-
-[[package]]
-name = "js-sys"
-version = "0.3.106"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5"
-dependencies = [
- "cfg-if",
- "futures-util",
- "wasm-bindgen",
-]
-
-[[package]]
-name = "libc"
-version = "0.2.191"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a"
-
-[[package]]
-name = "libloading"
-version = "0.8.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55"
-dependencies = [
- "cfg-if",
- "windows-link",
-]
-
-[[package]]
-name = "libloading"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "754ca22de805bb5744484a5b151a9e1a8e837d5dc232c2d7d8c2e3492edc8b60"
-dependencies = [
- "cfg-if",
- "windows-link",
-]
-
-[[package]]
-name = "libm"
-version = "0.2.16"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
-
-[[package]]
-name = "lock_api"
-version = "0.4.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965"
-dependencies = [
- "scopeguard",
-]
-
-[[package]]
-name = "log"
-version = "0.4.34"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6"
-
-[[package]]
-name = "luminamoon-dataloader"
-version = "1.1.0-post.3"
-dependencies = [
- "arrow-array",
- "arrow-ipc",
- "arrow-schema",
- "crc32fast",
- "hashbrown",
- "parking_lot",
- "parquet",
- "pyo3",
- "rayon",
- "serde",
- "serde_json",
- "thiserror",
- "xxhash-rust",
-]
-
-[[package]]
-name = "luminamoon-optimizer"
-version = "1.1.0-post.3"
-dependencies = [
- "luminamoon-dataloader",
- "luminav-core",
- "luminav-dispatch",
- "luminav-simd",
- "luminav-triton-aot",
- "luminav-vulkan",
- "pyo3",
- "rayon",
- "thiserror",
-]
-
-[[package]]
-name = "luminav-core"
-version = "1.1.0-post.3"
-dependencies = [
- "thiserror",
-]
-
-[[package]]
-name = "luminav-dispatch"
-version = "1.1.0-post.3"
-dependencies = [
- "luminav-core",
- "luminav-simd",
- "luminav-triton-aot",
- "luminav-vulkan",
- "thiserror",
-]
-
-[[package]]
-name = "luminav-simd"
-version = "1.1.0-post.3"
-dependencies = [
- "luminav-core",
- "rayon",
-]
-
-[[package]]
-name = "luminav-triton-aot"
-version = "1.1.0-post.3"
-dependencies = [
- "libloading 0.9.0",
- "luminav-core",
- "serde",
- "serde_json",
- "thiserror",
-]
-
-[[package]]
-name = "luminav-vulkan"
-version = "1.1.0-post.3"
-dependencies = [
- "ash",
- "luminav-core",
- "thiserror",
-]
-
-[[package]]
-name = "lz4_flex"
-version = "0.14.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ecbdfe44b1bd960b68170b417450a628c43f7cf56bb3c5317e61cb230ee7f226"
-dependencies = [
- "twox-hash",
-]
-
-[[package]]
-name = "memchr"
-version = "2.8.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
-
-[[package]]
-name = "num-bigint"
-version = "0.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "93e7820bc0a80a0238e650327316f929ba18d5be054b647490a3a6a339f3e7c0"
-dependencies = [
- "num-integer",
- "num-traits",
-]
-
-[[package]]
-name = "num-complex"
-version = "0.4.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495"
-dependencies = [
- "num-traits",
-]
-
-[[package]]
-name = "num-integer"
-version = "0.1.47"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b"
-dependencies = [
- "num-traits",
-]
-
-[[package]]
-name = "num-traits"
-version = "0.2.19"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
-dependencies = [
- "autocfg",
- "libm",
-]
-
-[[package]]
-name = "once_cell"
-version = "1.21.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
-
-[[package]]
-name = "parking_lot"
-version = "0.12.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a"
-dependencies = [
- "lock_api",
- "parking_lot_core",
-]
-
-[[package]]
-name = "parking_lot_core"
-version = "0.9.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1"
-dependencies = [
- "cfg-if",
- "libc",
- "redox_syscall",
- "smallvec",
- "windows-link",
-]
-
-[[package]]
-name = "parquet"
-version = "60.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "806eaa0f1c0436dcb43bee8e6a82bf2efff90521a60172f8281cb9ef577b43a4"
-dependencies = [
- "ahash",
- "arrow-array",
- "arrow-buffer",
- "arrow-data",
- "arrow-ipc",
- "arrow-schema",
- "arrow-select",
- "base64",
- "brotli",
- "bytes",
- "chrono",
- "flate2",
- "half",
- "hashbrown",
- "lz4_flex",
- "num-bigint",
- "num-integer",
- "num-traits",
- "seq-macro",
- "simdutf8",
- "snap",
- "twox-hash",
- "zstd",
-]
-
-[[package]]
-name = "pin-project-lite"
-version = "0.2.17"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"
-
-[[package]]
-name = "pkg-config"
-version = "0.3.34"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548"
-
-[[package]]
-name = "portable-atomic"
-version = "1.15.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85"
-
-[[package]]
-name = "proc-macro2"
-version = "1.0.107"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
-dependencies = [
- "unicode-ident",
-]
-
-[[package]]
-name = "pyo3"
-version = "0.30.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "229ca13b8a15bb0d1487b81ab975bfcd97513416499a8312011b6036e42e0de4"
-dependencies = [
- "libc",
- "once_cell",
- "portable-atomic",
- "pyo3-build-config",
- "pyo3-ffi",
- "pyo3-macros",
-]
-
-[[package]]
-name = "pyo3-build-config"
-version = "0.30.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "70130e2b6dd1ee589a8788026a429561ca5c40fa7a2fa7914c2282eddae24700"
-dependencies = [
- "target-lexicon",
-]
-
-[[package]]
-name = "pyo3-ffi"
-version = "0.30.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5e7ff73c555c286d212d4ce7e5fbda845a973c4f61bd5f160f5f2c3c2a04d1c4"
-dependencies = [
- "libc",
- "pyo3-build-config",
-]
-
-[[package]]
-name = "pyo3-macros"
-version = "0.30.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c46d2e5862c7bb8207e3b639f713dda3c499933444784154c11d013c5a691b60"
-dependencies = [
- "proc-macro2",
- "pyo3-macros-backend",
- "quote",
- "syn 3.0.9",
-]
-
-[[package]]
-name = "pyo3-macros-backend"
-version = "0.30.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "38b89f066f2ae72a2aef52d1537ca2fcd104440ebfadee06e97aa90738a25b9b"
-dependencies = [
- "heck",
- "proc-macro2",
- "quote",
- "syn 3.0.9",
-]
-
-[[package]]
-name = "quote"
-version = "1.0.47"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
-dependencies = [
- "proc-macro2",
-]
-
-[[package]]
-name = "r-efi"
-version = "5.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
-
-[[package]]
-name = "r-efi"
-version = "6.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
-
-[[package]]
-name = "rayon"
-version = "1.12.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d"
-dependencies = [
- "either",
- "rayon-core",
-]
-
-[[package]]
-name = "rayon-core"
-version = "1.13.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91"
-dependencies = [
- "crossbeam-deque",
- "crossbeam-utils",
-]
-
-[[package]]
-name = "redox_syscall"
-version = "0.5.18"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d"
-dependencies = [
- "bitflags",
-]
-
-[[package]]
-name = "rustc_version"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92"
-dependencies = [
- "semver",
-]
-
-[[package]]
-name = "rustversion"
-version = "1.0.23"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f"
-
-[[package]]
-name = "scopeguard"
-version = "1.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
-
-[[package]]
-name = "semver"
-version = "1.0.28"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd"
-
-[[package]]
-name = "seq-macro"
-version = "0.3.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc"
-
-[[package]]
-name = "serde"
-version = "1.0.229"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
-dependencies = [
- "serde_core",
- "serde_derive",
-]
-
-[[package]]
-name = "serde_core"
-version = "1.0.229"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
-dependencies = [
- "serde_derive",
-]
-
-[[package]]
-name = "serde_derive"
-version = "1.0.229"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 3.0.9",
-]
-
-[[package]]
-name = "serde_json"
-version = "1.0.154"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6"
-dependencies = [
- "itoa",
- "memchr",
- "serde",
- "serde_core",
- "zmij",
-]
-
-[[package]]
-name = "shlex"
-version = "2.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
-
-[[package]]
-name = "simdutf8"
-version = "0.1.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e"
-
-[[package]]
-name = "slab"
-version = "0.4.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5"
-
-[[package]]
-name = "smallvec"
-version = "1.16.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b3dc8af474f516a851ff4bd12db780f948b9250ad37211e4eec0bccea54e01b"
-
-[[package]]
-name = "snap"
-version = "1.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "199905e6153d6405f9728fe44daace35f8f837bbf830bb6e85fbd5828709a886"
-
-[[package]]
-name = "syn"
-version = "2.0.119"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
-dependencies = [
- "proc-macro2",
- "quote",
- "unicode-ident",
-]
-
-[[package]]
-name = "syn"
-version = "3.0.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b"
-dependencies = [
- "proc-macro2",
- "quote",
- "unicode-ident",
-]
-
-[[package]]
-name = "target-lexicon"
-version = "0.13.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
-
-[[package]]
-name = "thiserror"
-version = "2.0.21"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e"
-dependencies = [
- "thiserror-impl",
-]
-
-[[package]]
-name = "thiserror-impl"
-version = "2.0.21"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 3.0.9",
-]
-
-[[package]]
-name = "tiny-keccak"
-version = "2.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237"
-dependencies = [
- "crunchy",
-]
-
-[[package]]
-name = "twox-hash"
-version = "2.1.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "86a801b3cea342a06d468c8710662aa29e5e05e4f5c0d62f00bbb7f2ad7941c2"
-
-[[package]]
-name = "unicode-ident"
-version = "1.0.27"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563"
-
-[[package]]
-name = "version_check"
-version = "0.9.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
-
-[[package]]
-name = "wasi"
-version = "0.11.1+wasi-snapshot-preview1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
-
-[[package]]
-name = "wasip2"
-version = "1.0.4+wasi-0.2.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487"
-dependencies = [
- "wit-bindgen",
-]
-
-[[package]]
-name = "wasm-bindgen"
-version = "0.2.129"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409"
-dependencies = [
- "cfg-if",
- "once_cell",
- "rustversion",
- "wasm-bindgen-macro",
- "wasm-bindgen-shared",
-]
-
-[[package]]
-name = "wasm-bindgen-macro"
-version = "0.2.129"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535"
-dependencies = [
- "quote",
- "wasm-bindgen-macro-support",
-]
-
-[[package]]
-name = "wasm-bindgen-macro-support"
-version = "0.2.129"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7"
-dependencies = [
- "bumpalo",
- "proc-macro2",
- "quote",
- "syn 3.0.9",
- "wasm-bindgen-shared",
-]
-
-[[package]]
-name = "wasm-bindgen-shared"
-version = "0.2.129"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6"
-dependencies = [
- "unicode-ident",
-]
-
-[[package]]
-name = "windows-core"
-version = "0.62.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb"
-dependencies = [
- "windows-implement",
- "windows-interface",
- "windows-link",
- "windows-result",
- "windows-strings",
-]
-
-[[package]]
-name = "windows-implement"
-version = "0.60.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.119",
-]
-
-[[package]]
-name = "windows-interface"
-version = "0.59.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.119",
-]
-
-[[package]]
-name = "windows-link"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
-
-[[package]]
-name = "windows-result"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5"
-dependencies = [
- "windows-link",
-]
-
-[[package]]
-name = "windows-strings"
-version = "0.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091"
-dependencies = [
- "windows-link",
-]
-
-[[package]]
-name = "wit-bindgen"
-version = "0.57.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e"
-
-[[package]]
-name = "xxhash-rust"
-version = "0.8.19"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "550a2b930b62486a393c52d5c3b84bff264b28aa437ed64694d31e93b1757af7"
-
-[[package]]
-name = "zerocopy"
-version = "0.8.63"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e5fe1f8f1b06191a00962174c61aa5005e0bb391a6d80d07e24d115c01a92ed8"
-dependencies = [
- "zerocopy-derive",
-]
-
-[[package]]
-name = "zerocopy-derive"
-version = "0.8.63"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "863ad3ac83293fb4d740aedbfdc9240dd8d1a50c1099acd76ce80ce7c7230c7f"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.119",
-]
-
-[[package]]
-name = "zlib-rs"
-version = "0.6.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112"
-
-[[package]]
-name = "zmij"
-version = "1.0.23"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
-
-[[package]]
-name = "zstd"
-version = "0.14.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "db3ae9bf90af6e40197bef767eeefc46ad6397420fe873830bd85e5fbbfb3c70"
-dependencies = [
- "zstd-safe",
-]
-
-[[package]]
-name = "zstd-safe"
-version = "8.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ded520f16b29e1a9b771c722221b2b0a0398c2316d9bb3c42b458a5414d7434b"
-dependencies = [
- "zstd-sys",
-]
-
-[[package]]
-name = "zstd-sys"
-version = "2.1.2+zstd.1.5.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "481ada9c543b2323cbb7bbdbb5249177fbf0188ed959ed2c15e4b22421c84ed3"
-dependencies = [
- "cc",
- "pkg-config",
-]
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 4
+
+[[package]]
+name = "ahash"
+version = "0.8.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75"
+dependencies = [
+ "cfg-if",
+ "const-random",
+ "getrandom 0.3.4",
+ "once_cell",
+ "version_check",
+ "zerocopy",
+]
+
+[[package]]
+name = "alloc-no-stdlib"
+version = "3.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c2fb6cfd47bf496ff64095c20eaba0c201404ee38714d4142fcfa1dc334fcc7a"
+
+[[package]]
+name = "alloc-stdlib"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b5c1865780388bfa186411ab5f247819487fc4864c6e9c3106611fa347586e1"
+dependencies = [
+ "alloc-no-stdlib",
+]
+
+[[package]]
+name = "allocator-api2"
+version = "0.2.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
+
+[[package]]
+name = "android_system_properties"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "arrow-array"
+version = "60.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0f3d22a41d3df78b71df0b685af219ae3a94188ab98d6fb29c958eb55c3247f4"
+dependencies = [
+ "ahash",
+ "arrow-buffer",
+ "arrow-data",
+ "arrow-schema",
+ "chrono",
+ "half",
+ "hashbrown",
+ "num-complex",
+ "num-integer",
+ "num-traits",
+]
+
+[[package]]
+name = "arrow-buffer"
+version = "60.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7bd46a530bc1a8e763102fc36c1dc0f51f3ffa150c78e7f5658ccb54b1458ade"
+dependencies = [
+ "bytes",
+ "half",
+ "num-bigint",
+ "num-traits",
+]
+
+[[package]]
+name = "arrow-cmp"
+version = "60.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1913e57a64fbf2c4502fa6d54b547d6a1ae59e3146cbc15321d47468c937abd1"
+dependencies = [
+ "arrow-array",
+ "arrow-buffer",
+ "arrow-schema",
+]
+
+[[package]]
+name = "arrow-data"
+version = "60.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "415b7fbc4cf56b080bb5677a4ae74211b5c79e571ed37a1d547633ec07f3b43b"
+dependencies = [
+ "arrow-buffer",
+ "arrow-schema",
+ "half",
+ "num-integer",
+ "num-traits",
+]
+
+[[package]]
+name = "arrow-ipc"
+version = "60.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "45f3e766bccf5a0cb57a5de52a4cba450dea5171547c2cb5939b486d08887cba"
+dependencies = [
+ "arrow-array",
+ "arrow-buffer",
+ "arrow-data",
+ "arrow-schema",
+ "arrow-select",
+ "flatbuffers",
+]
+
+[[package]]
+name = "arrow-schema"
+version = "60.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "de2c8e714b466404cdc1189e89f892426efea614d938e16c15518c340cd43ba3"
+
+[[package]]
+name = "arrow-select"
+version = "60.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a5af3f6656c5f2c6944f93d9a357a7f204c93d5f349385a44cf46c4a26fbcf30"
+dependencies = [
+ "ahash",
+ "arrow-array",
+ "arrow-buffer",
+ "arrow-cmp",
+ "arrow-data",
+ "arrow-schema",
+ "num-traits",
+]
+
+[[package]]
+name = "ash"
+version = "0.38.0+1.3.281"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0bb44936d800fea8f016d7f2311c6a4f97aebd5dc86f09906139ec848cf3a46f"
+dependencies = [
+ "libloading 0.8.9",
+]
+
+[[package]]
+name = "autocfg"
+version = "1.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
+
+[[package]]
+name = "base64"
+version = "0.23.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5"
+
+[[package]]
+name = "bitflags"
+version = "2.13.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06"
+
+[[package]]
+name = "brotli"
+version = "9.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f8b851b75c23ca7873623d612fe49bd1989aeb03d08fb9432187eb253d3d4c6b"
+dependencies = [
+ "alloc-no-stdlib",
+ "alloc-stdlib",
+ "brotli-decompressor",
+]
+
+[[package]]
+name = "brotli-decompressor"
+version = "6.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "941cd9bd4ddab83cb46fa5a2d428f1c857b24ac78cb876cf7beb710840934bd7"
+dependencies = [
+ "alloc-no-stdlib",
+ "alloc-stdlib",
+]
+
+[[package]]
+name = "bumpalo"
+version = "3.20.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
+
+[[package]]
+name = "bytes"
+version = "1.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
+
+[[package]]
+name = "cc"
+version = "1.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6651c9ed80effdc7db0ff72512157f901af5e3549e341e24b1dd4887d836d838"
+dependencies = [
+ "find-msvc-tools",
+ "jobserver",
+ "libc",
+ "shlex",
+]
+
+[[package]]
+name = "cfg-if"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600"
+
+[[package]]
+name = "chrono"
+version = "0.4.45"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327"
+dependencies = [
+ "iana-time-zone",
+ "num-traits",
+ "windows-link",
+]
+
+[[package]]
+name = "const-random"
+version = "0.1.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359"
+dependencies = [
+ "const-random-macro",
+]
+
+[[package]]
+name = "const-random-macro"
+version = "0.1.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e"
+dependencies = [
+ "getrandom 0.2.17",
+ "once_cell",
+ "tiny-keccak",
+]
+
+[[package]]
+name = "core-foundation-sys"
+version = "0.8.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
+
+[[package]]
+name = "crc32fast"
+version = "1.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78"
+dependencies = [
+ "cfg-if",
+]
+
+[[package]]
+name = "crossbeam-deque"
+version = "0.8.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a"
+dependencies = [
+ "crossbeam-epoch",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-epoch"
+version = "0.9.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d"
+dependencies = [
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-utils"
+version = "0.8.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6"
+
+[[package]]
+name = "crunchy"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
+
+[[package]]
+name = "either"
+version = "1.19.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be"
+
+[[package]]
+name = "equivalent"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826"
+
+[[package]]
+name = "find-msvc-tools"
+version = "0.1.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484"
+
+[[package]]
+name = "flatbuffers"
+version = "25.12.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3"
+dependencies = [
+ "bitflags",
+ "rustc_version",
+]
+
+[[package]]
+name = "flate2"
+version = "1.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb"
+dependencies = [
+ "zlib-rs",
+]
+
+[[package]]
+name = "foldhash"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
+
+[[package]]
+name = "futures-core"
+version = "0.3.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e"
+
+[[package]]
+name = "futures-task"
+version = "0.3.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd"
+
+[[package]]
+name = "futures-util"
+version = "0.3.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc"
+dependencies = [
+ "futures-core",
+ "futures-task",
+ "pin-project-lite",
+ "slab",
+]
+
+[[package]]
+name = "getrandom"
+version = "0.2.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "wasi",
+]
+
+[[package]]
+name = "getrandom"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "r-efi 5.3.0",
+ "wasip2",
+]
+
+[[package]]
+name = "getrandom"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "r-efi 6.0.0",
+]
+
+[[package]]
+name = "half"
+version = "2.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b"
+dependencies = [
+ "cfg-if",
+ "crunchy",
+ "num-traits",
+ "zerocopy",
+]
+
+[[package]]
+name = "hashbrown"
+version = "0.17.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
+dependencies = [
+ "allocator-api2",
+ "equivalent",
+ "foldhash",
+]
+
+[[package]]
+name = "heck"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
+
+[[package]]
+name = "iana-time-zone"
+version = "0.1.65"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470"
+dependencies = [
+ "android_system_properties",
+ "core-foundation-sys",
+ "iana-time-zone-haiku",
+ "js-sys",
+ "log",
+ "wasm-bindgen",
+ "windows-core",
+]
+
+[[package]]
+name = "iana-time-zone-haiku"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
+dependencies = [
+ "cc",
+]
+
+[[package]]
+name = "itoa"
+version = "1.0.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
+
+[[package]]
+name = "jobserver"
+version = "0.1.35"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3"
+dependencies = [
+ "getrandom 0.4.3",
+ "libc",
+]
+
+[[package]]
+name = "js-sys"
+version = "0.3.106"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5"
+dependencies = [
+ "cfg-if",
+ "futures-util",
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "libc"
+version = "0.2.191"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a"
+
+[[package]]
+name = "libloading"
+version = "0.8.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55"
+dependencies = [
+ "cfg-if",
+ "windows-link",
+]
+
+[[package]]
+name = "libloading"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "754ca22de805bb5744484a5b151a9e1a8e837d5dc232c2d7d8c2e3492edc8b60"
+dependencies = [
+ "cfg-if",
+ "windows-link",
+]
+
+[[package]]
+name = "libm"
+version = "0.2.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
+
+[[package]]
+name = "lock_api"
+version = "0.4.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965"
+dependencies = [
+ "scopeguard",
+]
+
+[[package]]
+name = "log"
+version = "0.4.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6"
+
+[[package]]
+name = "luminamoon-dataloader"
+version = "1.1.0-post.3"
+dependencies = [
+ "arrow-array",
+ "arrow-ipc",
+ "arrow-schema",
+ "crc32fast",
+ "hashbrown",
+ "parking_lot",
+ "parquet",
+ "pyo3",
+ "rayon",
+ "serde",
+ "serde_json",
+ "thiserror",
+ "xxhash-rust",
+]
+
+[[package]]
+name = "luminamoon-optimizer"
+version = "1.1.0-post.3"
+dependencies = [
+ "luminamoon-dataloader",
+ "luminav-core",
+ "luminav-dispatch",
+ "luminav-simd",
+ "luminav-triton-aot",
+ "luminav-vulkan",
+ "pyo3",
+ "rayon",
+ "thiserror",
+]
+
+[[package]]
+name = "luminav-core"
+version = "1.1.0-post.3"
+dependencies = [
+ "thiserror",
+]
+
+[[package]]
+name = "luminav-dispatch"
+version = "1.1.0-post.3"
+dependencies = [
+ "luminav-core",
+ "luminav-simd",
+ "luminav-triton-aot",
+ "luminav-vulkan",
+ "thiserror",
+]
+
+[[package]]
+name = "luminav-simd"
+version = "1.1.0-post.3"
+dependencies = [
+ "luminav-core",
+ "rayon",
+]
+
+[[package]]
+name = "luminav-triton-aot"
+version = "1.1.0-post.3"
+dependencies = [
+ "libloading 0.9.0",
+ "luminav-core",
+ "serde",
+ "serde_json",
+ "thiserror",
+]
+
+[[package]]
+name = "luminav-vulkan"
+version = "1.1.0-post.3"
+dependencies = [
+ "ash",
+ "luminav-core",
+ "thiserror",
+]
+
+[[package]]
+name = "lz4_flex"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ecbdfe44b1bd960b68170b417450a628c43f7cf56bb3c5317e61cb230ee7f226"
+dependencies = [
+ "twox-hash",
+]
+
+[[package]]
+name = "memchr"
+version = "2.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
+
+[[package]]
+name = "num-bigint"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93e7820bc0a80a0238e650327316f929ba18d5be054b647490a3a6a339f3e7c0"
+dependencies = [
+ "num-integer",
+ "num-traits",
+]
+
+[[package]]
+name = "num-complex"
+version = "0.4.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495"
+dependencies = [
+ "num-traits",
+]
+
+[[package]]
+name = "num-integer"
+version = "0.1.47"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b"
+dependencies = [
+ "num-traits",
+]
+
+[[package]]
+name = "num-traits"
+version = "0.2.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
+dependencies = [
+ "autocfg",
+ "libm",
+]
+
+[[package]]
+name = "once_cell"
+version = "1.21.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
+
+[[package]]
+name = "parking_lot"
+version = "0.12.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a"
+dependencies = [
+ "lock_api",
+ "parking_lot_core",
+]
+
+[[package]]
+name = "parking_lot_core"
+version = "0.9.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "redox_syscall",
+ "smallvec",
+ "windows-link",
+]
+
+[[package]]
+name = "parquet"
+version = "60.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "806eaa0f1c0436dcb43bee8e6a82bf2efff90521a60172f8281cb9ef577b43a4"
+dependencies = [
+ "ahash",
+ "arrow-array",
+ "arrow-buffer",
+ "arrow-data",
+ "arrow-ipc",
+ "arrow-schema",
+ "arrow-select",
+ "base64",
+ "brotli",
+ "bytes",
+ "chrono",
+ "flate2",
+ "half",
+ "hashbrown",
+ "lz4_flex",
+ "num-bigint",
+ "num-integer",
+ "num-traits",
+ "seq-macro",
+ "simdutf8",
+ "snap",
+ "twox-hash",
+ "zstd",
+]
+
+[[package]]
+name = "pin-project-lite"
+version = "0.2.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"
+
+[[package]]
+name = "pkg-config"
+version = "0.3.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548"
+
+[[package]]
+name = "portable-atomic"
+version = "1.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85"
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.107"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "pyo3"
+version = "0.30.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "229ca13b8a15bb0d1487b81ab975bfcd97513416499a8312011b6036e42e0de4"
+dependencies = [
+ "libc",
+ "once_cell",
+ "portable-atomic",
+ "pyo3-build-config",
+ "pyo3-ffi",
+ "pyo3-macros",
+]
+
+[[package]]
+name = "pyo3-build-config"
+version = "0.30.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "70130e2b6dd1ee589a8788026a429561ca5c40fa7a2fa7914c2282eddae24700"
+dependencies = [
+ "target-lexicon",
+]
+
+[[package]]
+name = "pyo3-ffi"
+version = "0.30.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5e7ff73c555c286d212d4ce7e5fbda845a973c4f61bd5f160f5f2c3c2a04d1c4"
+dependencies = [
+ "libc",
+ "pyo3-build-config",
+]
+
+[[package]]
+name = "pyo3-macros"
+version = "0.30.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c46d2e5862c7bb8207e3b639f713dda3c499933444784154c11d013c5a691b60"
+dependencies = [
+ "proc-macro2",
+ "pyo3-macros-backend",
+ "quote",
+ "syn 3.0.9",
+]
+
+[[package]]
+name = "pyo3-macros-backend"
+version = "0.30.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "38b89f066f2ae72a2aef52d1537ca2fcd104440ebfadee06e97aa90738a25b9b"
+dependencies = [
+ "heck",
+ "proc-macro2",
+ "quote",
+ "syn 3.0.9",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.47"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "r-efi"
+version = "5.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
+
+[[package]]
+name = "r-efi"
+version = "6.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
+
+[[package]]
+name = "rayon"
+version = "1.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d"
+dependencies = [
+ "either",
+ "rayon-core",
+]
+
+[[package]]
+name = "rayon-core"
+version = "1.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91"
+dependencies = [
+ "crossbeam-deque",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "redox_syscall"
+version = "0.5.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d"
+dependencies = [
+ "bitflags",
+]
+
+[[package]]
+name = "rustc_version"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92"
+dependencies = [
+ "semver",
+]
+
+[[package]]
+name = "rustversion"
+version = "1.0.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f"
+
+[[package]]
+name = "scopeguard"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
+
+[[package]]
+name = "semver"
+version = "1.0.28"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd"
+
+[[package]]
+name = "seq-macro"
+version = "0.3.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc"
+
+[[package]]
+name = "serde"
+version = "1.0.229"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
+dependencies = [
+ "serde_core",
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_core"
+version = "1.0.229"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.229"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 3.0.9",
+]
+
+[[package]]
+name = "serde_json"
+version = "1.0.154"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6"
+dependencies = [
+ "itoa",
+ "memchr",
+ "serde",
+ "serde_core",
+ "zmij",
+]
+
+[[package]]
+name = "shlex"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
+
+[[package]]
+name = "simdutf8"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e"
+
+[[package]]
+name = "slab"
+version = "0.4.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5"
+
+[[package]]
+name = "smallvec"
+version = "1.16.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b3dc8af474f516a851ff4bd12db780f948b9250ad37211e4eec0bccea54e01b"
+
+[[package]]
+name = "snap"
+version = "1.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "199905e6153d6405f9728fe44daace35f8f837bbf830bb6e85fbd5828709a886"
+
+[[package]]
+name = "syn"
+version = "2.0.119"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "syn"
+version = "3.0.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "target-lexicon"
+version = "0.13.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
+
+[[package]]
+name = "thiserror"
+version = "2.0.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e"
+dependencies = [
+ "thiserror-impl",
+]
+
+[[package]]
+name = "thiserror-impl"
+version = "2.0.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 3.0.9",
+]
+
+[[package]]
+name = "tiny-keccak"
+version = "2.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237"
+dependencies = [
+ "crunchy",
+]
+
+[[package]]
+name = "twox-hash"
+version = "2.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "86a801b3cea342a06d468c8710662aa29e5e05e4f5c0d62f00bbb7f2ad7941c2"
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.27"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563"
+
+[[package]]
+name = "version_check"
+version = "0.9.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
+
+[[package]]
+name = "wasi"
+version = "0.11.1+wasi-snapshot-preview1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
+
+[[package]]
+name = "wasip2"
+version = "1.0.4+wasi-0.2.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487"
+dependencies = [
+ "wit-bindgen",
+]
+
+[[package]]
+name = "wasm-bindgen"
+version = "0.2.129"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409"
+dependencies = [
+ "cfg-if",
+ "once_cell",
+ "rustversion",
+ "wasm-bindgen-macro",
+ "wasm-bindgen-shared",
+]
+
+[[package]]
+name = "wasm-bindgen-macro"
+version = "0.2.129"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535"
+dependencies = [
+ "quote",
+ "wasm-bindgen-macro-support",
+]
+
+[[package]]
+name = "wasm-bindgen-macro-support"
+version = "0.2.129"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7"
+dependencies = [
+ "bumpalo",
+ "proc-macro2",
+ "quote",
+ "syn 3.0.9",
+ "wasm-bindgen-shared",
+]
+
+[[package]]
+name = "wasm-bindgen-shared"
+version = "0.2.129"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "windows-core"
+version = "0.62.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb"
+dependencies = [
+ "windows-implement",
+ "windows-interface",
+ "windows-link",
+ "windows-result",
+ "windows-strings",
+]
+
+[[package]]
+name = "windows-implement"
+version = "0.60.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
+]
+
+[[package]]
+name = "windows-interface"
+version = "0.59.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
+]
+
+[[package]]
+name = "windows-link"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
+
+[[package]]
+name = "windows-result"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5"
+dependencies = [
+ "windows-link",
+]
+
+[[package]]
+name = "windows-strings"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091"
+dependencies = [
+ "windows-link",
+]
+
+[[package]]
+name = "wit-bindgen"
+version = "0.57.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e"
+
+[[package]]
+name = "xxhash-rust"
+version = "0.8.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "550a2b930b62486a393c52d5c3b84bff264b28aa437ed64694d31e93b1757af7"
+
+[[package]]
+name = "zerocopy"
+version = "0.8.63"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e5fe1f8f1b06191a00962174c61aa5005e0bb391a6d80d07e24d115c01a92ed8"
+dependencies = [
+ "zerocopy-derive",
+]
+
+[[package]]
+name = "zerocopy-derive"
+version = "0.8.63"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "863ad3ac83293fb4d740aedbfdc9240dd8d1a50c1099acd76ce80ce7c7230c7f"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
+]
+
+[[package]]
+name = "zlib-rs"
+version = "0.6.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112"
+
+[[package]]
+name = "zmij"
+version = "1.0.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
+
+[[package]]
+name = "zstd"
+version = "0.14.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "db3ae9bf90af6e40197bef767eeefc46ad6397420fe873830bd85e5fbbfb3c70"
+dependencies = [
+ "zstd-safe",
+]
+
+[[package]]
+name = "zstd-safe"
+version = "8.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ded520f16b29e1a9b771c722221b2b0a0398c2316d9bb3c42b458a5414d7434b"
+dependencies = [
+ "zstd-sys",
+]
+
+[[package]]
+name = "zstd-sys"
+version = "2.1.2+zstd.1.5.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "481ada9c543b2323cbb7bbdbb5249177fbf0188ed959ed2c15e4b22421c84ed3"
+dependencies = [
+ "cc",
+ "pkg-config",
+]

Cargo.toml

@@ -1,39 +1,39 @@
-[workspace]
-resolver = "3"
-members = [
-    "optimizer",
-    "optimizer/crates/*",
-    "data",
-]
-
-[workspace.package]
-version = "1.1.0-post.3"
-edition = "2024"
-license = "Apache-2.0"
-authors = ["Silver Moon <leafmoon.idn@gmail.com>", "Lumina Moon (cloverx-id) Contributors"]
-repository = "https://github.com/cloverx-id/luminamoon"
-homepage = "https://github.com/cloverx-id/luminamoon"
-
-[workspace.dependencies]
-pyo3 = { version = "0.30.1", features = ["abi3-py310", "abi3t"] }
-ash = "0.38.0"
-libloading = "0.9.0"
-rayon = "1.12.0"
-thiserror = "2.0.21"
-serde = { version = "1.0.229", features = ["derive"] }
-serde_json = "1.0.154"
-parquet = "60.1.0"
-arrow-array = "60.1.0"
-arrow-schema = "60.1.0"
-arrow-ipc = "60.1.0"
-xxhash-rust = { version = "0.8.19", features = ["xxh3"] }
-crc32fast = "1.5.2"
-hashbrown = "0.17.1"
-parking_lot = "0.12.5"
-
-[profile.release]
-opt-level = 3
-lto = "fat"
-codegen-units = 16
-panic = "abort"
-strip = true
+[workspace]
+resolver = "3"
+members = [
+    "optimizer",
+    "optimizer/crates/*",
+    "data",
+]
+
+[workspace.package]
+version = "1.1.0-post.3"
+edition = "2024"
+license = "Apache-2.0"
+authors = ["Silver Moon <leafmoon.idn@gmail.com>", "Lumina Moon (cloverx-id) Contributors"]
+repository = "https://github.com/cloverx-id/luminamoon"
+homepage = "https://github.com/cloverx-id/luminamoon"
+
+[workspace.dependencies]
+pyo3 = { version = "0.30.1", features = ["abi3-py310", "abi3t"] }
+ash = "0.38.0"
+libloading = "0.9.0"
+rayon = "1.12.0"
+thiserror = "2.0.21"
+serde = { version = "1.0.229", features = ["derive"] }
+serde_json = "1.0.154"
+parquet = "60.1.0"
+arrow-array = "60.1.0"
+arrow-schema = "60.1.0"
+arrow-ipc = "60.1.0"
+xxhash-rust = { version = "0.8.19", features = ["xxh3"] }
+crc32fast = "1.5.2"
+hashbrown = "0.17.1"
+parking_lot = "0.12.5"
+
+[profile.release]
+opt-level = 3
+lto = "fat"
+codegen-units = 16
+panic = "abort"
+strip = true

LICENSE

@@ -1,13 +1,13 @@
-Copyright 2026 Silver Moon (cloverxion) and Lumina Moon (cloverx-id) Contributors
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    https://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
+Copyright 2026 Silver Moon (cloverxion) and Lumina Moon (cloverx-id) Contributors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

README.md

@@ -1,334 +1,334 @@
----
-license: apache-2.0
-library_name: pytorch
-language:
-- en
-tags:
-- pytorch
-- optimizer
-- custom-optimizer
-- luminamoon
-- luminav
-- clm
-- td-clm
-- lso-eval
-- rust
-- simd
-- avx512
-- vulkan
-- triton
-- memory-efficient
-- stochastic-rounding
-- low-vram
-- bf16
-- fp16
-- cuda
-- rocm
-- xpu
-- llm
-- transformer
-- deep-learning
-- research
----
-
-# Loomy
-
-<p align="left">
-  <a href="https://doi.org/10.5281/zenodo.23973326"><img src="https://zenodo.org/badge/DOI/10.5281/zenodo.23973326.svg" alt="DOI"></a>
-  <a href="https://pypi.org/project/luminamoon/"><img src="https://img.shields.io/pypi/v/luminamoon.svg?color=4f46e5" alt="PyPI version"></a>
-  <a href="https://github.com/cloverx-id/luminamoon/actions/workflows/ci.yml"><img src="https://github.com/cloverx-id/luminamoon/actions/workflows/ci.yml/badge.svg" alt="CI Status"></a>
-  <a href="eval/README.md"><img src="https://img.shields.io/badge/LSO--Eval-Verified-10b981" alt="LSO-Eval"></a>
-  <a href="eval/LSO_Eval.pdf"><img src="https://img.shields.io/badge/Eval_Docs-PDF-b31b1b" alt="Eval Docs"></a>
-  <a href="https://cloverx-id.github.io/luminamoon/"><img src="https://img.shields.io/badge/docs-GitHub_Pages-06b6d4" alt="Documentation"></a>
-  <a href="https://pypi.org/project/luminamoon/"><img src="https://img.shields.io/badge/python-3.10%20%E2%86%92%203.15t%20(free--threaded)-10b981" alt="Python Versions"></a>
-  <a href="LICENSE"><img src="https://img.shields.io/badge/license-Apache--2.0-blue" alt="License"></a>
-</p>
-
-**Loomy (`luminamoon`)** provides low-precision numerical optimization, rational neural network layers, and dataset text preprocessing utilities developed by **Silver Moon ([@cloverxion](https://github.com/cloverxion))** and **Lumina Moon ([@cloverx-id](https://github.com/cloverx-id)) contributors**.
-
-Built with a modular Rust and Python architecture, `luminamoon` provides components for CPU/GPU parameter updates and text dataset preprocessing:
-
-> **LSO-Eval Evaluation & Verification Notebook**: [**Open in Google Colab**](https://colab.research.google.com/drive/1UepDza5Wm3KRNooCA2Bn69GIeKB21a4a?usp=sharing) — *Stage II–IV benchmarks executed on `LuminaV v1.5.3` under native `torch.bfloat16` (`centered_defect=False`).*
-
-## Projects & Components
-
-| Project / Component | Path | Status | Description | Documentation |
-| :--- | :--- | :--- | :--- | :--- |
-| **LuminaV Optimizer** | [`optimizer/`](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/optimizer/README.md) | `Active` (Math `v1.6.0`) | LuminaV v1.6.0 optimizer implementation with CPU SIMD (AVX-512/AVX2), Vulkan UMA, Triton AOT/JIT kernels, Triangle Family (`buffer="T1"`, `"T2"`), optional contiguous bucket fusion, and Triton JIT pre-warmup. | [Optimizer Guide](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/optimizer/README.md) |
-| **LuminaMoon Data Engine** | [`data/`](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/data/README.md) | `Active` (`v1.1.0`) | Modular text preprocessing library: AVX-512/AVX2/NEON SIMD text normalization, dual-layout StringTensor (CPU StringView & GPU contiguous buffers), Shannon entropy filtering, MinHash/XXH3-64 deduplication, PII masking, and 128B Ping-Pong checkpoint recovery. | [Data Engine Guide](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/data/README.md) |
-| **LSO-Eval** | [`eval/`](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/eval/README.md) | `Active` | Algorithm-agnostic optimizer evaluation & verification: Z3 SMT solver checks, 2D trajectory benchmarks, numerical parity tests, and Two-Class 4B LLM & Vision evaluation. | [LSO-Eval Guide](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/eval/README.md) |
-| **Changelogs** | [`changelogs/`](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/changelogs/README.md) | `Active` | Version histories separating Software Package releases (`v1.0.0`–`v1.1.0`) and LuminaV Algorithm Math versions (`v1.0.0`–`v1.6.0`). | [Changelogs](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/changelogs/README.md) |
-| **Documentation** | [`docs/`](https://cloverx-id.github.io/luminamoon/) | `Active` | Documentation index, developer guides (`docs/optimize/`, `docs/dataloader/`), and dependency version notes. | [Docs Index](https://cloverx-id.github.io/luminamoon/) |
-| **Ra-Family Architecture** | [`python/luminamoon/nn/`](https://huggingface.co/cloverx-id/LuminaV-Optimizer/tree/main/python/luminamoon/nn) | `Active` (`v1.1.0`) | Rational, tanh-free & square-root-free neural components: RaNorm, RaBatchNorm2d (with Conv2d weight folding), RaTanh, RaGELU, PosRaGLU (bounded gating in `(0, 1)`), and RaAttention with Cayley-Padé rational softmax. Triton GPU and Rust AVX2/AVX-512 SIMD acceleration. | [Ra-Family Guide](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/docs/nn/README.md) |
-| **Python Package** | [`python/luminamoon/`](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/python/luminamoon/__init__.py) | `Active` (`v1.1.0`) | Unified free-threaded (`abi3t`) Python namespace exposing LuminaV Optimizer, DataStreamLoader, Ra-Family neural architecture, and PyTorch/NumPy/HuggingFace integration. | [API Reference](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/docs/en/03_configuration_api.md) |
-| **Theoretical References** | [`references/`](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/references/README.md) | `Active` | Mathematical formulas, component mapping, literature comparisons, and BibTeX citations for LuminaV, Ra-Family, and Data Engine. | [References Guide](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/references/README.md) |
-
-## Quickstart
-
-### Installation
-
-Install the compiled package wheel from PyPI:
-
-```bash
-pip install luminamoon
-```
-
-Or build from source using Maturin and the Rust 2024 toolchain:
-
-```bash
-git clone https://github.com/cloverx-id/luminamoon.git
-cd luminamoon
-pip install maturin
-maturin develop --release
-```
-
-### Basic Usage (LuminaV Optimizer)
-
-```python
-import numpy as np
-from luminamoon import LuminaV
-
-# Initialize parameters
-weights = np.random.randn(64, 10).astype(np.float32)
-biases = np.zeros(10, dtype=np.float32)
-
-# Instantiate LuminaV optimizer (uses default hyperparameters)
-optimizer = LuminaV([weights, biases], lr=8e-4, weight_decay=0.08, buffer=2)
-
-# Execute optimization step with explicit gradients
-grad_w = np.random.randn(64, 10).astype(np.float32)
-grad_b = np.random.randn(10, dtype=np.float32)
-optimizer.step(grads=[grad_w, grad_b])
-```
-
-For PyTorch deep learning workflows, see the [PyTorch Integration Guide](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/docs/en/01_getting_started.md#quickstart-pytorch-integration).
-
-### LuminaV Hyperparameter Quick Reference
-
-All default values below are exposed in IDE autocompletion and `LuminaV.__init__` docstrings:
-
-| Parameter | Type | Default | Description |
-| :--- | :--- | :--- | :--- |
-| `lr` | `float` | `8e-4` (`0.0008`) | Learning rate (`η`). |
-| `betas` | `Tuple[float, float]` | `(0.9, 0.999)` | Exponential moving average factors `(β₁, β₂)` for first-moment momentum and second-moment variance. |
-| `eps` | `float` | `1e-8` | Numerical stability constant (`ε`) in the adaptive denominator. |
-| `weight_decay` | `float` | `0.08` (`8e-2`) | Decoupled weight decay coefficient. |
-| `tau` | `float` | `0.8` | Denominator temperature scaling factor (`τ`) for 2-buffer variance damping. |
-| `lam` | `float` | `0.3` | Triangle-defect penalty weight (`λ`) on `\|d_t\| = \|g_t - m_{t-1}\|` in 1-Buffer TD-CLM (`buffer=1`) and Triangle Family (`buffer="T1"`, `"T2"`). |
-| `cautious` | `bool` | `True` | Enables sign-aligned cautious update masking and lagged EMA scaling. |
-| `cautious_clamp_min` | `float` | `0.5` | Lower bound in `(0.0, 1.0]` for cautious alignment scaling. |
-| `buffer` | `int \| str` | `2` | State mode: `2` (`"dual"`, `"clm"`) for 2-Buffer CLM, `1` (`"single"`, `"td-clm"`, `"tdclm"`) for 1-Buffer TD-CLM, `"T2"` for 2-Buffer Triangle Family (`tanh`-free), or `"T1"` for 1-Buffer Triangle Family (`tanh`-free). |
-| `stochastic_rounding` | `bool` | `True` | Applies stateless 32-bit MurmurHash3 stochastic rounding when writing to `bfloat16` / `float16` weights. |
-| `bound` | `bool` | `True` | Enables adaptive update bounding. |
-| `bound_type` | `str \| int` | `"coordinate"` | Clipping mode: `"coordinate"` (element-wise clip) or `"radial"` (global norm ratio). |
-| `bound_ratio` | `float` | `0.05` | Maximum update magnitude relative to parameter scale. |
-| `master_weights` | `str \| bool` | `"none"` | Mixed-precision master weight mode for `bfloat16`/`float16`: `"none"` (`False`), `"semi"`, or `"full"` (`True`). |
-| `execution` | `str \| int` | `"auto"` | Hardware backend: `"auto"`, `"triton"`, `"vulkan"`, `"simd"`, or `"scalar"`. |
-| `fused_single_pass` | `bool` | `True` | Fuses parameter update and scalar reduction statistics into a single kernel pass. |
-| `fuse_tensors` | `bool` | `False` | Coalesces parameters into contiguous 1D memory buffers to reduce kernel launch latency on fragmented architectures (e.g. Vision models / ViT). Not recommended for large LLMs or distributed FSDP training. |
-| `bucket_cap_mb` | `float` | `500.0` | Maximum memory capacity per contiguous bucket chunk in megabytes when `fuse_tensors=True`. Partitions large models into bounded chunks. Emits a `UserWarning` if explicitly specified when `fuse_tensors=False`. |
-| `prewarm_jit` | `bool` | `True` | Pre-compiles Triton GPU kernels on 64-element dummy buffers (`lr=0.0`) during initialization to avoid first-step JIT compilation delay. |
-| `ema_decay` | `str \| float` | `"auto"` | EMA decay for lagged cautious/radial scalars (`"auto"` resolves to `0.80` for `buffer=2`/`"T2"` and `0.95` for `buffer=1`/`"T1"`). |
-| `fsdp_sync` | `bool` | `False` | Synchronizes global reduction scratch buffers across `torch.distributed` FSDP shards. |
-| `seed` | `int` | `1337` | Deterministic PRNG seed for stateless stochastic rounding. |
-| `version` | `Optional[str]` | `"1.6.0"` | Selects the active LuminaV mathematical specification version. |
-
-### Software vs. Algorithm Versioning (`v1.1.0` / Math `v1.6.0`)
-
-`luminamoon` maintains two distinct, decoupled version lineages (documented in [`changelogs/`](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/changelogs/README.md)):
-
-1. **Software Package Version (`v1.1.0`)**:
-   - Tracks the monorepo distribution, CPython FFI bindings, Rust SIMD crates, neural architecture (`luminamoon.nn`), and Data Engine utilities on PyPI (`pip install luminamoon`).
-   - To reproduce or execute older software package releases in Python, install the desired release directly via standard package management (`pip install luminamoon==1.0.9`) or check out official git release tags.
-   - See [`changelogs/SOFTWARE_CHANGELOG.md`](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/changelogs/SOFTWARE_CHANGELOG.md).
-
-2. **LuminaV Algorithm Math Version (`v1.6.0`)**:
-   - Tracks the mathematical optimization formulas, convergence envelopes, and hardware kernels of the **LuminaV Optimizer** independently from pip package updates.
-   - Four mathematical versions are supported: `1.6.0` (default with Triangle Family `T1`/`T2` and Padé rational bounds), `1.5.5` (bucket fusion and Triton pre-warmup), `1.5.4` (Bitwise-BF16 state promotion), and `1.5.3` (single-buffer TD-CLM).
-   - Features **strict capability enforcement**: locking to an older mathematical version (such as `version="1.5.4"`) clearly rejects features introduced in later versions (such as `buffer="T2"` or `fuse_tensors=True`), helping ensure exact reproducibility for benchmarks and publications.
-   - See [`changelogs/LUMINAV_ALGORITHM_CHANGELOG.md`](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/CHANGELOG.md).
-
-```python
-import luminamoon
-from luminamoon import LuminaV
-import luminamoon.data as lmd
-
-# 1. Inspect Software Package Version vs. Algorithm Math Version
-print(luminamoon.__version__)    # "1.1.0" (Software package release)
-print(lmd.get_version())         # "1.1.0" (Data Engine package module)
-print(LuminaV.ALGORITHM_MATH)    # "1.6.0" (Algorithm mathematical version)
-print(LuminaV.ALGORITHM_SPEC)    # "1.6.0" (Backward-compatible alias)
-print(LuminaV.getVersion())      # "1.6.0"
-
-# 2. Inspect capability matrix of any supported specification
-caps = LuminaV.get_capabilities("1.5.5")
-print(caps["allowed_buffers"])       # (1, 2)
-print(caps["supports_triangle_family"]) # False (introduced in 1.6.0)
-
-# 3. Lock algorithm version for scientific checkpoint reproducibility
-opt = LuminaV([weights, biases], lr=8e-4, version="1.5.5")
-opt.setVersion("1.5.5")          # Instance-level version lock
-LuminaV.setVersion("1.5.5")      # Class-level default specification
-```
-
-### Basic Usage (LuminaMoon Data Engine: Fused Text Preprocessing Pipeline)
-
-```python
-from luminamoon.data import FusedPipeline, StringTensor, create_resumable_collate, ResumablePipeline
-
-# 1. Fused single-pass text preprocessing in Rust
-pipeline = FusedPipeline(
-    trim_whitespace=True,
-    normalize_newlines=True,  # AVX-512 / AVX2 / NEON SIMD
-    min_chars=10,
-    min_entropy=2.0,
-)
-string_tensor = pipeline(["  sample 1\r\n", "sample 2 with proper prose text"])
-
-# 2. Export to Apache Arrow, Polars, and PyTorch buffers
-polars_series = string_tensor.to_polars()
-arrow_array = string_tensor.to_arrow()
-offsets_tensor, payload_tensor = string_tensor.to_torch_gpu(device="cuda")
-
-# 3. Stream dataset with background prefetching & 128B Ping-Pong checkpointing
-pipe = ResumablePipeline(
-    source=raw_data,
-    fused_pipeline=pipeline,
-    batch_size=64,
-    prefetch_batches=4,
-    checkpoint_path="./cache/checkpoint.meta",
-)
-for batch in pipe:
-    # Training step...
-    pipe.commit()  # Saves Dual-Slot A/B checkpoint and deduplication WAL
-```
-
-For detailed architecture, Shannon entropy filtering, MinHash deduplication, and PyTorch `collate_fn` adapters, see the [Data Engine Guide](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/data/README.md).
-
-### Basic Usage (Ra-Family Rational Neural Architecture)
-
-```python
-import torch
-from luminamoon.nn import RaNorm, PosRaGLU, RaAttention
-
-# 1. Square-root-free token normalization (L1-mean)
-norm = RaNorm(512)
-x = torch.randn(2, 64, 512)
-x_norm = norm(x)
-
-# 2. Rational gated linear unit with strictly positive bounds in (0, 1)
-mlp = PosRaGLU(512, hidden_features=2048)
-x_ffn = mlp(x_norm)
-
-# 3. Cayley-Padé rational multi-head attention (SFU-free softmax)
-attn = RaAttention(dim=512, num_heads=8, qk_norm=True, causal=True)
-out = attn(x_norm)
-```
-
-For complete architecture, parameter folding for Conv2d, and standalone NumPy usage, see the [Ra-Family Guide](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/docs/nn/README.md).
-
-## Verification & CI/CD
-
-All pull requests and releases are tested across Linux, Windows, and macOS:
-
-```bash
-# Run static analysis across the Rust workspace
-cargo clippy --workspace --all-targets -- -D warnings
-
-# Run Rust workspace unit and parity tests (Optimizer + Data Engine)
-cargo test --workspace
-
-# Run Python static type check
-npx pyright python/luminamoon
-
-# Run standalone NumPy, PyTorch, and Data Engine tests
-python optimizer/tests/test_standalone_numpy.py
-python optimizer/tests/test_end_to_end_training.py
-python -B tests/test_data_modular.py
-python -B tests/test_gpu_vs_cpu_strings.py
-
-# Run LSO-Eval verification
-python -m eval
-```
-
-## Developer Resources
-
-For contributors working on **Loomy (`luminamoon`)**:
-
-- **Changelogs (Software & Algorithm)**: [`changelogs/`](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/changelogs/README.md)
-- **Optimizer Modular Documentation**: [`docs/optimize/`](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/docs/optimize/README.md)
-- **Data Engine Modular Documentation**: [`data/`](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/data/README.md)
-- **Optimizer Library Notes**: [`docs/library_audit/`](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/docs/library_audit/README.md)
-- **Data Standalone Library Notes**: [`data/docs/library_audit/`](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/data/docs/library_audit/README.md)
-- **Data Engine Architecture & User Guide**: [`data/README.md`](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/data/README.md)
-- **User Documentation**: [`docs/en/`](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/docs/en/README.md)
-
-## Theoretical References & Citation
-
-LuminaMoon's algorithms are built upon published mathematical principles, including Maligranda's triangle inequality envelopes, rational neural networks, and decoupled optimization mechanics.
-
-For the full component mapping, literature comparisons, and BibTeX entries, see [`references/README.md`](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/references/README.md) and [`references/bibliography.bib`](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/references/bibliography.bib).
-
-To cite this repository in academic work:
-
-```bibtex
-@misc{loomy2026,
-  author = {Silver Moon and {Lumina Moon contributors}},
-  title = {Loomy: Low-Precision Numerical Optimization, Rational Neural Architecture, and Data Processing Repository},
-  year = {2026},
-  publisher = {GitHub},
-  howpublished = {\url{https://github.com/cloverx-id/luminamoon}},
-  doi = {10.5281/zenodo.23973326},
-  note = {Loomy repository}
-}
-```
-
-## Authors & Maintenance
-
-**Loomy (`luminamoon`)** is developed and maintained by:
-
-- **Silver Moon ([@cloverxion](https://github.com/cloverxion))**
-- **Lumina Moon ([@cloverx-id](https://github.com/cloverx-id)) contributors**
-
-## License
-
-Licensed under the Apache License, Version 2.0. See [LICENSE](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/LICENSE) for details.
-
----
-
-## LSO-Eval: Verification & Pretraining Benchmarks (`Qwen3.5-4B-Base`)
-
-Summary of verification checks and pretraining benchmarks evaluated on **LuminaV (`v1.5.3`)** using the **LSO-Eval** suite, covering invariant checks (Z3 SMT solver and precision tests), 2D test functions, and pretraining runs on `Qwen3.5-4B-Base` in `bfloat16`:
-
-> **Benchmark Note (`LuminaV v1.5.3`)**: Results below were evaluated on `LuminaV v1.5.3` under `torch.bfloat16` (`centered_defect=False`). In the evaluation loader, records were sampled by joint `(source, domain)` pairs; all candidate and baseline optimizers shared the same deterministic stream schedule.
-
-### Stage IV-B: Context 2048 (20,000 Steps)
-
-<div align="center">
-  <img src="https://huggingface.co/cloverx-id/LuminaV-Optimizer/resolve/main/eval/artifacts/v1.5.3/stage4b_lso_eval_results.png" alt="LSO-Eval Stage IV-B Qwen3.5-4B (ctx=2048)" width="100%" />
-</div>
-
-| Optimizer Configuration | Class | Final EMA | Val PPL | ΔPPL | VRAM<sub>peak</sub> | Max R<sub>t,l</sub> | Step / Opt Time |
-| :--- | :--- | :---: | :---: | :---: | :---: | :---: | :---: |
-| **`LuminaV_Buffer2_Dual` (`8e-4`)** | Class I (Dual) | **`4.5549`** | **`89.14`** | **`-43.43`** | `43.89 GB` | `0.0189` | `297.1 / 48.31 ms` |
-| **`LuminaV_Buffer1_TDCLM_3e4` (`3e-4`)** | Class II (Single) | `4.9173` | `165.13` | `+32.56` | **`35.97 GB`** | `0.0087` | **`283.5 / 36.99 ms`** |
-| `AdamW` (Native BF16, `3e-4`) | Baseline | `4.7621` | `132.57` | `+0.00` | `43.89 GB` | `0.0037` | `288.0 / 41.48 ms` |
-| `AdamW_FP32_Master` (`3e-4`) | Master Ref | `4.8116` | `124.43` | `-8.14` | `80.11 GB` | `0.0045` | `370.1 / 123.33 ms` |
-
-### Stage IV-A: Context 512 (4,800 Steps, Grad Accum 8x)
-
-<div align="center">
-  <img src="https://huggingface.co/cloverx-id/LuminaV-Optimizer/resolve/main/eval/artifacts/v1.5.3/stage4a_lso_eval_results.png" alt="LSO-Eval Stage IV-A Qwen3.5-4B (ctx=512)" width="100%" />
-</div>
-
-| Optimizer Configuration | Class | Final EMA | Val PPL | ΔPPL | VRAM<sub>peak</sub> | Max R<sub>t,l</sub> | Step / Opt Time |
-| :--- | :--- | :---: | :---: | :---: | :---: | :---: | :---: |
-| **`LuminaV_Buffer2_Dual` (`8e-4`)** | Class I (Dual) | **`4.7621`** | **`141.96`** | **`-39.21`** | `36.75 GB` | `0.0047` | `841.6 / 48.37 ms` |
-| **`LuminaV_Buffer1_TDCLM_3e4` (`3e-4`)** | Class II (Single) | `5.2569` | `195.58` | `+14.41` | **`28.81 GB`** | `0.0120` | **`829.4 / 36.77 ms`** |
-| **`LuminaV_Buffer1_TDCLM` (`8e-4`)** | Class II (Single) | `5.2953` | `221.07` | `+39.90` | **`28.81 GB`** | `0.0191` | **`830.7 / 36.46 ms`** |
-| `AdamW` (Native BF16, `3e-4`) | Baseline | `5.4497` | `181.17` | `+0.00` | `36.75 GB` | `0.0034` | `835.4 / 41.58 ms` |
-| `AdamW_FP32_Master` (`3e-4`) | Master Ref | `5.0234` | `153.78` | `-27.39` | `75.36 GB` | `0.0040` | `919.5 / 123.35 ms` |
-
-- **[LSO-Eval Formal Specification Manuscript (`eval/LSO_Eval.pdf`)](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/eval/LSO_Eval.pdf)**
-- **[Full LSO-Eval Verification Summary & Suite Guide (`eval/README.md`)](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/eval/README.md)**
-- **[Interactive Reproduction Notebook in Google Colab](https://colab.research.google.com/drive/1UepDza5Wm3KRNooCA2Bn69GIeKB21a4a?usp=sharing)**
+---
+license: apache-2.0
+library_name: pytorch
+language:
+- en
+tags:
+- pytorch
+- optimizer
+- custom-optimizer
+- luminamoon
+- luminav
+- clm
+- td-clm
+- lso-eval
+- rust
+- simd
+- avx512
+- vulkan
+- triton
+- memory-efficient
+- stochastic-rounding
+- low-vram
+- bf16
+- fp16
+- cuda
+- rocm
+- xpu
+- llm
+- transformer
+- deep-learning
+- research
+---
+
+# Loomy
+
+<p align="left">
+  <a href="https://doi.org/10.5281/zenodo.23973326"><img src="https://zenodo.org/badge/DOI/10.5281/zenodo.23973326.svg" alt="DOI"></a>
+  <a href="https://pypi.org/project/luminamoon/"><img src="https://img.shields.io/pypi/v/luminamoon.svg?color=4f46e5" alt="PyPI version"></a>
+  <a href="https://github.com/cloverx-id/luminamoon/actions/workflows/ci.yml"><img src="https://github.com/cloverx-id/luminamoon/actions/workflows/ci.yml/badge.svg" alt="CI Status"></a>
+  <a href="eval/README.md"><img src="https://img.shields.io/badge/LSO--Eval-Verified-10b981" alt="LSO-Eval"></a>
+  <a href="eval/LSO_Eval.pdf"><img src="https://img.shields.io/badge/Eval_Docs-PDF-b31b1b" alt="Eval Docs"></a>
+  <a href="https://cloverx-id.github.io/luminamoon/"><img src="https://img.shields.io/badge/docs-GitHub_Pages-06b6d4" alt="Documentation"></a>
+  <a href="https://pypi.org/project/luminamoon/"><img src="https://img.shields.io/badge/python-3.10%20%E2%86%92%203.15t%20(free--threaded)-10b981" alt="Python Versions"></a>
+  <a href="LICENSE"><img src="https://img.shields.io/badge/license-Apache--2.0-blue" alt="License"></a>
+</p>
+
+**Loomy (`luminamoon`)** provides low-precision numerical optimization, rational neural network layers, and dataset text preprocessing utilities developed by **Silver Moon ([@cloverxion](https://github.com/cloverxion))** and **Lumina Moon ([@cloverx-id](https://github.com/cloverx-id)) contributors**.
+
+Built with a modular Rust and Python architecture, `luminamoon` provides components for CPU/GPU parameter updates and text dataset preprocessing:
+
+> **LSO-Eval Evaluation & Verification Notebook**: [**Open in Google Colab**](https://colab.research.google.com/drive/1UepDza5Wm3KRNooCA2Bn69GIeKB21a4a?usp=sharing) — *Stage II–IV benchmarks executed on `LuminaV v1.5.3` under native `torch.bfloat16` (`centered_defect=False`).*
+
+## Projects & Components
+
+| Project / Component | Path | Status | Description | Documentation |
+| :--- | :--- | :--- | :--- | :--- |
+| **LuminaV Optimizer** | [`optimizer/`](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/optimizer/README.md) | `Active` (Math `v1.6.0`) | LuminaV v1.6.0 optimizer implementation with CPU SIMD (AVX-512/AVX2), Vulkan UMA, Triton AOT/JIT kernels, Triangle Family (`buffer="T1"`, `"T2"`), optional contiguous bucket fusion, and Triton JIT pre-warmup. | [Optimizer Guide](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/optimizer/README.md) |
+| **LuminaMoon Data Engine** | [`data/`](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/data/README.md) | `Active` (`v1.1.0`) | Modular text preprocessing library: AVX-512/AVX2/NEON SIMD text normalization, dual-layout StringTensor (CPU StringView & GPU contiguous buffers), Shannon entropy filtering, MinHash/XXH3-64 deduplication, PII masking, and 128B Ping-Pong checkpoint recovery. | [Data Engine Guide](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/data/README.md) |
+| **LSO-Eval** | [`eval/`](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/eval/README.md) | `Active` | Algorithm-agnostic optimizer evaluation & verification: Z3 SMT solver checks, 2D trajectory benchmarks, numerical parity tests, and Two-Class 4B LLM & Vision evaluation. | [LSO-Eval Guide](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/eval/README.md) |
+| **Changelogs** | [`changelogs/`](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/changelogs/README.md) | `Active` | Version histories separating Software Package releases (`v1.0.0`–`v1.1.0`) and LuminaV Algorithm Math versions (`v1.0.0`–`v1.6.0`). | [Changelogs](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/changelogs/README.md) |
+| **Documentation** | [`docs/`](https://cloverx-id.github.io/luminamoon/) | `Active` | Documentation index, developer guides (`docs/optimize/`, `docs/dataloader/`), and dependency version notes. | [Docs Index](https://cloverx-id.github.io/luminamoon/) |
+| **Ra-Family Architecture** | [`python/luminamoon/nn/`](https://huggingface.co/cloverx-id/LuminaV-Optimizer/tree/main/python/luminamoon/nn) | `Active` (`v1.1.0`) | Rational, tanh-free & square-root-free neural components: RaNorm, RaBatchNorm2d (with Conv2d weight folding), RaTanh, RaGELU, PosRaGLU (bounded gating in `(0, 1)`), and RaAttention with Cayley-Padé rational softmax. Triton GPU and Rust AVX2/AVX-512 SIMD acceleration. | [Ra-Family Guide](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/docs/nn/README.md) |
+| **Python Package** | [`python/luminamoon/`](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/python/luminamoon/__init__.py) | `Active` (`v1.1.0`) | Unified free-threaded (`abi3t`) Python namespace exposing LuminaV Optimizer, DataStreamLoader, Ra-Family neural architecture, and PyTorch/NumPy/HuggingFace integration. | [API Reference](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/docs/en/03_configuration_api.md) |
+| **Theoretical References** | [`references/`](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/references/README.md) | `Active` | Mathematical formulas, component mapping, literature comparisons, and BibTeX citations for LuminaV, Ra-Family, and Data Engine. | [References Guide](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/references/README.md) |
+
+## Quickstart
+
+### Installation
+
+Install the compiled package wheel from PyPI:
+
+```bash
+pip install luminamoon
+```
+
+Or build from source using Maturin and the Rust 2024 toolchain:
+
+```bash
+git clone https://github.com/cloverx-id/luminamoon.git
+cd luminamoon
+pip install maturin
+maturin develop --release
+```
+
+### Basic Usage (LuminaV Optimizer)
+
+```python
+import numpy as np
+from luminamoon import LuminaV
+
+# Initialize parameters
+weights = np.random.randn(64, 10).astype(np.float32)
+biases = np.zeros(10, dtype=np.float32)
+
+# Instantiate LuminaV optimizer (uses default hyperparameters)
+optimizer = LuminaV([weights, biases], lr=8e-4, weight_decay=0.08, buffer=2)
+
+# Execute optimization step with explicit gradients
+grad_w = np.random.randn(64, 10).astype(np.float32)
+grad_b = np.random.randn(10, dtype=np.float32)
+optimizer.step(grads=[grad_w, grad_b])
+```
+
+For PyTorch deep learning workflows, see the [PyTorch Integration Guide](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/docs/en/01_getting_started.md#quickstart-pytorch-integration).
+
+### LuminaV Hyperparameter Quick Reference
+
+All default values below are exposed in IDE autocompletion and `LuminaV.__init__` docstrings:
+
+| Parameter | Type | Default | Description |
+| :--- | :--- | :--- | :--- |
+| `lr` | `float` | `8e-4` (`0.0008`) | Learning rate (`η`). |
+| `betas` | `Tuple[float, float]` | `(0.9, 0.999)` | Exponential moving average factors `(β₁, β₂)` for first-moment momentum and second-moment variance. |
+| `eps` | `float` | `1e-8` | Numerical stability constant (`ε`) in the adaptive denominator. |
+| `weight_decay` | `float` | `0.08` (`8e-2`) | Decoupled weight decay coefficient. |
+| `tau` | `float` | `0.8` | Denominator temperature scaling factor (`τ`) for 2-buffer variance damping. |
+| `lam` | `float` | `0.3` | Triangle-defect penalty weight (`λ`) on `\|d_t\| = \|g_t - m_{t-1}\|` in 1-Buffer TD-CLM (`buffer=1`) and Triangle Family (`buffer="T1"`, `"T2"`). |
+| `cautious` | `bool` | `True` | Enables sign-aligned cautious update masking and lagged EMA scaling. |
+| `cautious_clamp_min` | `float` | `0.5` | Lower bound in `(0.0, 1.0]` for cautious alignment scaling. |
+| `buffer` | `int \| str` | `2` | State mode: `2` (`"dual"`, `"clm"`) for 2-Buffer CLM, `1` (`"single"`, `"td-clm"`, `"tdclm"`) for 1-Buffer TD-CLM, `"T2"` for 2-Buffer Triangle Family (`tanh`-free), or `"T1"` for 1-Buffer Triangle Family (`tanh`-free). |
+| `stochastic_rounding` | `bool` | `True` | Applies stateless 32-bit MurmurHash3 stochastic rounding when writing to `bfloat16` / `float16` weights. |
+| `bound` | `bool` | `True` | Enables adaptive update bounding. |
+| `bound_type` | `str \| int` | `"coordinate"` | Clipping mode: `"coordinate"` (element-wise clip) or `"radial"` (global norm ratio). |
+| `bound_ratio` | `float` | `0.05` | Maximum update magnitude relative to parameter scale. |
+| `master_weights` | `str \| bool` | `"none"` | Mixed-precision master weight mode for `bfloat16`/`float16`: `"none"` (`False`), `"semi"`, or `"full"` (`True`). |
+| `execution` | `str \| int` | `"auto"` | Hardware backend: `"auto"`, `"triton"`, `"vulkan"`, `"simd"`, or `"scalar"`. |
+| `fused_single_pass` | `bool` | `True` | Fuses parameter update and scalar reduction statistics into a single kernel pass. |
+| `fuse_tensors` | `bool` | `False` | Coalesces parameters into contiguous 1D memory buffers to reduce kernel launch latency on fragmented architectures (e.g. Vision models / ViT). Not recommended for large LLMs or distributed FSDP training. |
+| `bucket_cap_mb` | `float` | `500.0` | Maximum memory capacity per contiguous bucket chunk in megabytes when `fuse_tensors=True`. Partitions large models into bounded chunks. Emits a `UserWarning` if explicitly specified when `fuse_tensors=False`. |
+| `prewarm_jit` | `bool` | `True` | Pre-compiles Triton GPU kernels on 64-element dummy buffers (`lr=0.0`) during initialization to avoid first-step JIT compilation delay. |
+| `ema_decay` | `str \| float` | `"auto"` | EMA decay for lagged cautious/radial scalars (`"auto"` resolves to `0.80` for `buffer=2`/`"T2"` and `0.95` for `buffer=1`/`"T1"`). |
+| `fsdp_sync` | `bool` | `False` | Synchronizes global reduction scratch buffers across `torch.distributed` FSDP shards. |
+| `seed` | `int` | `1337` | Deterministic PRNG seed for stateless stochastic rounding. |
+| `version` | `Optional[str]` | `"1.6.0"` | Selects the active LuminaV mathematical specification version. |
+
+### Software vs. Algorithm Versioning (`v1.1.0` / Math `v1.6.0`)
+
+`luminamoon` maintains two distinct, decoupled version lineages (documented in [`changelogs/`](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/changelogs/README.md)):
+
+1. **Software Package Version (`v1.1.0`)**:
+   - Tracks the monorepo distribution, CPython FFI bindings, Rust SIMD crates, neural architecture (`luminamoon.nn`), and Data Engine utilities on PyPI (`pip install luminamoon`).
+   - To reproduce or execute older software package releases in Python, install the desired release directly via standard package management (`pip install luminamoon==1.0.9`) or check out official git release tags.
+   - See [`changelogs/SOFTWARE_CHANGELOG.md`](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/changelogs/SOFTWARE_CHANGELOG.md).
+
+2. **LuminaV Algorithm Math Version (`v1.6.0`)**:
+   - Tracks the mathematical optimization formulas, convergence envelopes, and hardware kernels of the **LuminaV Optimizer** independently from pip package updates.
+   - Four mathematical versions are supported: `1.6.0` (default with Triangle Family `T1`/`T2` and Padé rational bounds), `1.5.5` (bucket fusion and Triton pre-warmup), `1.5.4` (Bitwise-BF16 state promotion), and `1.5.3` (single-buffer TD-CLM).
+   - Features **strict capability enforcement**: locking to an older mathematical version (such as `version="1.5.4"`) clearly rejects features introduced in later versions (such as `buffer="T2"` or `fuse_tensors=True`), helping ensure exact reproducibility for benchmarks and publications.
+   - See [`changelogs/LUMINAV_ALGORITHM_CHANGELOG.md`](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/CHANGELOG.md).
+
+```python
+import luminamoon
+from luminamoon import LuminaV
+import luminamoon.data as lmd
+
+# 1. Inspect Software Package Version vs. Algorithm Math Version
+print(luminamoon.__version__)    # "1.1.0" (Software package release)
+print(lmd.get_version())         # "1.1.0" (Data Engine package module)
+print(LuminaV.ALGORITHM_MATH)    # "1.6.0" (Algorithm mathematical version)
+print(LuminaV.ALGORITHM_SPEC)    # "1.6.0" (Backward-compatible alias)
+print(LuminaV.getVersion())      # "1.6.0"
+
+# 2. Inspect capability matrix of any supported specification
+caps = LuminaV.get_capabilities("1.5.5")
+print(caps["allowed_buffers"])       # (1, 2)
+print(caps["supports_triangle_family"]) # False (introduced in 1.6.0)
+
+# 3. Lock algorithm version for scientific checkpoint reproducibility
+opt = LuminaV([weights, biases], lr=8e-4, version="1.5.5")
+opt.setVersion("1.5.5")          # Instance-level version lock
+LuminaV.setVersion("1.5.5")      # Class-level default specification
+```
+
+### Basic Usage (LuminaMoon Data Engine: Fused Text Preprocessing Pipeline)
+
+```python
+from luminamoon.data import FusedPipeline, StringTensor, create_resumable_collate, ResumablePipeline
+
+# 1. Fused single-pass text preprocessing in Rust
+pipeline = FusedPipeline(
+    trim_whitespace=True,
+    normalize_newlines=True,  # AVX-512 / AVX2 / NEON SIMD
+    min_chars=10,
+    min_entropy=2.0,
+)
+string_tensor = pipeline(["  sample 1\r\n", "sample 2 with proper prose text"])
+
+# 2. Export to Apache Arrow, Polars, and PyTorch buffers
+polars_series = string_tensor.to_polars()
+arrow_array = string_tensor.to_arrow()
+offsets_tensor, payload_tensor = string_tensor.to_torch_gpu(device="cuda")
+
+# 3. Stream dataset with background prefetching & 128B Ping-Pong checkpointing
+pipe = ResumablePipeline(
+    source=raw_data,
+    fused_pipeline=pipeline,
+    batch_size=64,
+    prefetch_batches=4,
+    checkpoint_path="./cache/checkpoint.meta",
+)
+for batch in pipe:
+    # Training step...
+    pipe.commit()  # Saves Dual-Slot A/B checkpoint and deduplication WAL
+```
+
+For detailed architecture, Shannon entropy filtering, MinHash deduplication, and PyTorch `collate_fn` adapters, see the [Data Engine Guide](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/data/README.md).
+
+### Basic Usage (Ra-Family Rational Neural Architecture)
+
+```python
+import torch
+from luminamoon.nn import RaNorm, PosRaGLU, RaAttention
+
+# 1. Square-root-free token normalization (L1-mean)
+norm = RaNorm(512)
+x = torch.randn(2, 64, 512)
+x_norm = norm(x)
+
+# 2. Rational gated linear unit with strictly positive bounds in (0, 1)
+mlp = PosRaGLU(512, hidden_features=2048)
+x_ffn = mlp(x_norm)
+
+# 3. Cayley-Padé rational multi-head attention (SFU-free softmax)
+attn = RaAttention(dim=512, num_heads=8, qk_norm=True, causal=True)
+out = attn(x_norm)
+```
+
+For complete architecture, parameter folding for Conv2d, and standalone NumPy usage, see the [Ra-Family Guide](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/docs/nn/README.md).
+
+## Verification & CI/CD
+
+All pull requests and releases are tested across Linux, Windows, and macOS:
+
+```bash
+# Run static analysis across the Rust workspace
+cargo clippy --workspace --all-targets -- -D warnings
+
+# Run Rust workspace unit and parity tests (Optimizer + Data Engine)
+cargo test --workspace
+
+# Run Python static type check
+npx pyright python/luminamoon
+
+# Run standalone NumPy, PyTorch, and Data Engine tests
+python optimizer/tests/test_standalone_numpy.py
+python optimizer/tests/test_end_to_end_training.py
+python -B tests/test_data_modular.py
+python -B tests/test_gpu_vs_cpu_strings.py
+
+# Run LSO-Eval verification
+python -m eval
+```
+
+## Developer Resources
+
+For contributors working on **Loomy (`luminamoon`)**:
+
+- **Changelogs (Software & Algorithm)**: [`changelogs/`](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/changelogs/README.md)
+- **Optimizer Modular Documentation**: [`docs/optimize/`](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/docs/optimize/README.md)
+- **Data Engine Modular Documentation**: [`data/`](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/data/README.md)
+- **Optimizer Library Notes**: [`docs/library_audit/`](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/docs/library_audit/README.md)
+- **Data Standalone Library Notes**: [`data/docs/library_audit/`](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/data/docs/library_audit/README.md)
+- **Data Engine Architecture & User Guide**: [`data/README.md`](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/data/README.md)
+- **User Documentation**: [`docs/en/`](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/docs/en/README.md)
+
+## Theoretical References & Citation
+
+LuminaMoon's algorithms are built upon published mathematical principles, including Maligranda's triangle inequality envelopes, rational neural networks, and decoupled optimization mechanics.
+
+For the full component mapping, literature comparisons, and BibTeX entries, see [`references/README.md`](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/references/README.md) and [`references/bibliography.bib`](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/references/bibliography.bib).
+
+To cite this repository in academic work:
+
+```bibtex
+@misc{loomy2026,
+  author = {Silver Moon and {Lumina Moon contributors}},
+  title = {Loomy: Low-Precision Numerical Optimization, Rational Neural Architecture, and Data Processing Repository},
+  year = {2026},
+  publisher = {GitHub},
+  howpublished = {\url{https://github.com/cloverx-id/luminamoon}},
+  doi = {10.5281/zenodo.23973326},
+  note = {Loomy repository}
+}
+```
+
+## Authors & Maintenance
+
+**Loomy (`luminamoon`)** is developed and maintained by:
+
+- **Silver Moon ([@cloverxion](https://github.com/cloverxion))**
+- **Lumina Moon ([@cloverx-id](https://github.com/cloverx-id)) contributors**
+
+## License
+
+Licensed under the Apache License, Version 2.0. See [LICENSE](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/LICENSE) for details.
+
+---
+
+## LSO-Eval: Verification & Pretraining Benchmarks (`Qwen3.5-4B-Base`)
+
+Summary of verification checks and pretraining benchmarks evaluated on **LuminaV (`v1.5.3`)** using the **LSO-Eval** suite, covering invariant checks (Z3 SMT solver and precision tests), 2D test functions, and pretraining runs on `Qwen3.5-4B-Base` in `bfloat16`:
+
+> **Benchmark Note (`LuminaV v1.5.3`)**: Results below were evaluated on `LuminaV v1.5.3` under `torch.bfloat16` (`centered_defect=False`). In the evaluation loader, records were sampled by joint `(source, domain)` pairs; all candidate and baseline optimizers shared the same deterministic stream schedule.
+
+### Stage IV-B: Context 2048 (20,000 Steps)
+
+<div align="center">
+  <img src="https://huggingface.co/cloverx-id/LuminaV-Optimizer/resolve/main/eval/artifacts/v1.5.3/stage4b_lso_eval_results.png" alt="LSO-Eval Stage IV-B Qwen3.5-4B (ctx=2048)" width="100%" />
+</div>
+
+| Optimizer Configuration | Class | Final EMA | Val PPL | ΔPPL | VRAM<sub>peak</sub> | Max R<sub>t,l</sub> | Step / Opt Time |
+| :--- | :--- | :---: | :---: | :---: | :---: | :---: | :---: |
+| **`LuminaV_Buffer2_Dual` (`8e-4`)** | Class I (Dual) | **`4.5549`** | **`89.14`** | **`-43.43`** | `43.89 GB` | `0.0189` | `297.1 / 48.31 ms` |
+| **`LuminaV_Buffer1_TDCLM_3e4` (`3e-4`)** | Class II (Single) | `4.9173` | `165.13` | `+32.56` | **`35.97 GB`** | `0.0087` | **`283.5 / 36.99 ms`** |
+| `AdamW` (Native BF16, `3e-4`) | Baseline | `4.7621` | `132.57` | `+0.00` | `43.89 GB` | `0.0037` | `288.0 / 41.48 ms` |
+| `AdamW_FP32_Master` (`3e-4`) | Master Ref | `4.8116` | `124.43` | `-8.14` | `80.11 GB` | `0.0045` | `370.1 / 123.33 ms` |
+
+### Stage IV-A: Context 512 (4,800 Steps, Grad Accum 8x)
+
+<div align="center">
+  <img src="https://huggingface.co/cloverx-id/LuminaV-Optimizer/resolve/main/eval/artifacts/v1.5.3/stage4a_lso_eval_results.png" alt="LSO-Eval Stage IV-A Qwen3.5-4B (ctx=512)" width="100%" />
+</div>
+
+| Optimizer Configuration | Class | Final EMA | Val PPL | ΔPPL | VRAM<sub>peak</sub> | Max R<sub>t,l</sub> | Step / Opt Time |
+| :--- | :--- | :---: | :---: | :---: | :---: | :---: | :---: |
+| **`LuminaV_Buffer2_Dual` (`8e-4`)** | Class I (Dual) | **`4.7621`** | **`141.96`** | **`-39.21`** | `36.75 GB` | `0.0047` | `841.6 / 48.37 ms` |
+| **`LuminaV_Buffer1_TDCLM_3e4` (`3e-4`)** | Class II (Single) | `5.2569` | `195.58` | `+14.41` | **`28.81 GB`** | `0.0120` | **`829.4 / 36.77 ms`** |
+| **`LuminaV_Buffer1_TDCLM` (`8e-4`)** | Class II (Single) | `5.2953` | `221.07` | `+39.90` | **`28.81 GB`** | `0.0191` | **`830.7 / 36.46 ms`** |
+| `AdamW` (Native BF16, `3e-4`) | Baseline | `5.4497` | `181.17` | `+0.00` | `36.75 GB` | `0.0034` | `835.4 / 41.58 ms` |
+| `AdamW_FP32_Master` (`3e-4`) | Master Ref | `5.0234` | `153.78` | `-27.39` | `75.36 GB` | `0.0040` | `919.5 / 123.35 ms` |
+
+- **[LSO-Eval Formal Specification Manuscript (`eval/LSO_Eval.pdf`)](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/eval/LSO_Eval.pdf)**
+- **[Full LSO-Eval Verification Summary & Suite Guide (`eval/README.md`)](https://huggingface.co/cloverx-id/LuminaV-Optimizer/blob/main/eval/README.md)**
+- **[Interactive Reproduction Notebook in Google Colab](https://colab.research.google.com/drive/1UepDza5Wm3KRNooCA2Bn69GIeKB21a4a?usp=sharing)**

SECURITY.md

@@ -1,50 +1,50 @@
-# Security Policy
-
-## Supported Versions
-
-Security updates and bug fixes are maintained for the following versions of **LuminaMoon** (`luminamoon`) and the **LuminaV Optimizer** (`luminav`):
-
-| Component / Package | Version | Security Support Status |
-| :--- | :--- | :--- |
-| **`luminamoon`** (Rust and Python Package) | `>= 1.1.0` | Supported (Active) |
-| **`luminav`** (Algorithm Math) | `>= 1.6.0` | Supported (Active) |
-| **`luminamoon`** (Maintenance Line) | `1.0.8` – `1.0.9` | Supported (Maintenance) |
-| **`luminav`** (Maintenance Line) | `1.5.5` | Supported (Maintenance) |
-| **`luminamoon`** | `< 1.0.8` | Unsupported (Upgrade recommended) |
-| **`luminav`** | `< 1.5.5` | Unsupported (Upgrade recommended) |
-
----
-
-## Reporting a Vulnerability
-
-If you discover a security vulnerability, memory-safety issue in FFI/SIMD/Vulkan/Triton boundaries, or dependency flaw:
-
-1. **Private GitHub Security Advisory (Preferred)**:
-   Submit a confidential report through [GitHub Private Vulnerability Reporting](https://github.com/cloverx-id/luminamoon/security/advisories/new).
-2. **Direct Contact**:
-   Email **Silver Moon** (`@cloverxion`) directly at [`leafmoon.idn@gmail.com`](mailto:leafmoon.idn@gmail.com) with the subject prefix `[SECURITY][luminamoon]`.
-
-### Response & Disclosure Timeline
-- **Initial Acknowledgment**: Within 48 hours of report submission.
-- **Triage & Verification**: Within 5 business days.
-- **Fix & Disclosure**: Critical fixes are prioritized for release on GitHub, PyPI, and Hugging Face Hub alongside a published GitHub Security Advisory (GHSA / CVE).
-
-Please do not open public GitHub issues for undisclosed security vulnerabilities.
-
----
-
-## Automated Security Checks
-
-Every commit and pull request in `cloverx-id/luminamoon` is checked across automated workflows (`.github/workflows/security.yml`):
-
-1. **Static Analysis (GitHub CodeQL)**:
-   Automated dataflow analysis (`github/codeql-action@v4`) across Python modules and GitHub Actions workflows to catch injection, traversal, and unsafe deserialization issues.
-2. **Dependency & Supply-Chain Scanning (`tools/security_audit.py` & `pip-audit`)**:
-   Locked Rust crates in `Cargo.lock` and Python dependencies in `pyproject.toml` are checked against the RustSec Advisory Database, Open Source Vulnerabilities (OSV), PyPA, and the National Vulnerability Database.
-3. **Dependency Review Gate (`actions/dependency-review-action@v5`)**:
-   Blocks pull requests that introduce dependencies with known vulnerabilities.
-4. **Secret Scanning & Push Protection**:
-   GitHub Secret Scanning and Push Protection are active to prevent accidental credential commits.
-5. **Memory & FFI Checks**:
-   - Pointer alignment and contiguity verification across PyO3 (`abi3-py310` / `abi3t`) and NumPy/PyTorch buffers.
-   - Minimal dependency footprint: only actively required crates are locked in `Cargo.lock` (see [`docs/library_audit/`](docs/library_audit/README.md)).
+# Security Policy
+
+## Supported Versions
+
+Security updates and bug fixes are maintained for the following versions of **LuminaMoon** (`luminamoon`) and the **LuminaV Optimizer** (`luminav`):
+
+| Component / Package | Version | Security Support Status |
+| :--- | :--- | :--- |
+| **`luminamoon`** (Rust and Python Package) | `>= 1.1.0` | Supported (Active) |
+| **`luminav`** (Algorithm Math) | `>= 1.6.0` | Supported (Active) |
+| **`luminamoon`** (Maintenance Line) | `1.0.8` – `1.0.9` | Supported (Maintenance) |
+| **`luminav`** (Maintenance Line) | `1.5.5` | Supported (Maintenance) |
+| **`luminamoon`** | `< 1.0.8` | Unsupported (Upgrade recommended) |
+| **`luminav`** | `< 1.5.5` | Unsupported (Upgrade recommended) |
+
+---
+
+## Reporting a Vulnerability
+
+If you discover a security vulnerability, memory-safety issue in FFI/SIMD/Vulkan/Triton boundaries, or dependency flaw:
+
+1. **Private GitHub Security Advisory (Preferred)**:
+   Submit a confidential report through [GitHub Private Vulnerability Reporting](https://github.com/cloverx-id/luminamoon/security/advisories/new).
+2. **Direct Contact**:
+   Email **Silver Moon** (`@cloverxion`) directly at [`leafmoon.idn@gmail.com`](mailto:leafmoon.idn@gmail.com) with the subject prefix `[SECURITY][luminamoon]`.
+
+### Response & Disclosure Timeline
+- **Initial Acknowledgment**: Within 48 hours of report submission.
+- **Triage & Verification**: Within 5 business days.
+- **Fix & Disclosure**: Critical fixes are prioritized for release on GitHub, PyPI, and Hugging Face Hub alongside a published GitHub Security Advisory (GHSA / CVE).
+
+Please do not open public GitHub issues for undisclosed security vulnerabilities.
+
+---
+
+## Automated Security Checks
+
+Every commit and pull request in `cloverx-id/luminamoon` is checked across automated workflows (`.github/workflows/security.yml`):
+
+1. **Static Analysis (GitHub CodeQL)**:
+   Automated dataflow analysis (`github/codeql-action@v4`) across Python modules and GitHub Actions workflows to catch injection, traversal, and unsafe deserialization issues.
+2. **Dependency & Supply-Chain Scanning (`tools/security_audit.py` & `pip-audit`)**:
+   Locked Rust crates in `Cargo.lock` and Python dependencies in `pyproject.toml` are checked against the RustSec Advisory Database, Open Source Vulnerabilities (OSV), PyPA, and the National Vulnerability Database.
+3. **Dependency Review Gate (`actions/dependency-review-action@v5`)**:
+   Blocks pull requests that introduce dependencies with known vulnerabilities.
+4. **Secret Scanning & Push Protection**:
+   GitHub Secret Scanning and Push Protection are active to prevent accidental credential commits.
+5. **Memory & FFI Checks**:
+   - Pointer alignment and contiguity verification across PyO3 (`abi3-py310` / `abi3t`) and NumPy/PyTorch buffers.
+   - Minimal dependency footprint: only actively required crates are locked in `Cargo.lock` (see [`docs/library_audit/`](docs/library_audit/README.md)).

benchmarks/bench_data_processing.py

@@ -1,411 +1,411 @@
-"""LuminaMoon Data Processing Engine Benchmark.
-
-Compares Native Rust SIMD Accelerator (AVX-512 / AVX2 / Rayon) against
-Pure Python Fallbacks across micro-benchmarks and end-to-end pipelines.
-"""
-
-from __future__ import annotations
-
-import json
-import math
-import os
-import random
-import sys
-import tempfile
-import time
-from dataclasses import asdict, dataclass
-from pathlib import Path
-from typing import Any, Callable, Dict, List, Tuple
-
-# Ensure local python package is discoverable
-sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "python"))
-
-from luminamoon.optimizer import _luminamoon_optimizer_rs as rs
-from luminamoon.data import fallbacks as fb
-
-
-@dataclass
-class BenchmarkResult:
-    benchmark_name: str
-    workload_size: int
-    data_bytes: int
-    rust_duration_sec: float
-    rust_throughput_items_per_sec: float
-    rust_throughput_mb_per_sec: float
-    fallback_duration_sec: float
-    fallback_throughput_items_per_sec: float
-    fallback_throughput_mb_per_sec: float
-    speedup_ratio: float
-
-
-def generate_synthetic_dataset(num_records: int = 10000, seed: int = 42) -> Tuple[List[str], int]:
-    rng = random.Random(seed)
-    templates = [
-        "   The quick brown fox jumps over the lazy dog.\r\n   This is line 2 with carriage returns.\r\n",
-        "Deep learning optimization requires efficient gradient descent with momentum and adaptive learning rates.\n",
-        '{"text": "{\"key\": \"value\", \"nested\": [1, 2, 3]}", "meta": "test record"}\r\n',
-        "System security alert: detected unauthorized access attempt from 192.168.1.105 on port 443.\n",
-        "Contact user support at alice.walker@enterprise-domain.org or call +1-800-555-0199 for assistance.\r\n",
-        "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n",  # high repetition
-        "Short text.\n",
-        "Mathematics: Pade [13/6] approximant ensures subnormal FP16 RNE preservation without overflow.\n",
-        "High symbol noise: #$%@!&*()!@#$%^&*()_+{}|:<>?~`!@#$%^&*()_+{}|:<>?~`\n",
-        "Large document block: " + ("lorem ipsum dolor sit amet " * 20) + "\r\n",
-    ]
-    records = []
-    total_bytes = 0
-    for i in range(num_records):
-        base = templates[i % len(templates)]
-        noise = f" [doc_id={i}_{rng.randint(100, 999)}]"
-        rec = base + noise
-        records.append(rec)
-        total_bytes += len(rec.encode("utf-8"))
-    return records, total_bytes
-
-
-def run_timed_benchmark(
-    name: str,
-    rust_fn: Callable[[], Any],
-    fallback_fn: Callable[[], Any],
-    workload_size: int,
-    data_bytes: int,
-    warmup_runs: int = 2,
-    test_runs: int = 5,
-) -> BenchmarkResult:
-    # Warmup Rust
-    for _ in range(warmup_runs):
-        rust_fn()
-    # Timed Rust
-    rust_times = []
-    for _ in range(test_runs):
-        t0 = time.perf_counter_ns()
-        rust_fn()
-        t1 = time.perf_counter_ns()
-        rust_times.append((t1 - t0) / 1e9)
-    rust_dur = min(rust_times)  # best of N runs
-
-    # Warmup Fallback
-    for _ in range(warmup_runs):
-        fallback_fn()
-    # Timed Fallback
-    fallback_times = []
-    for _ in range(test_runs):
-        t0 = time.perf_counter_ns()
-        fallback_fn()
-        t1 = time.perf_counter_ns()
-        fallback_times.append((t1 - t0) / 1e9)
-    fallback_dur = min(fallback_times)
-
-    rust_items_per_sec = workload_size / max(rust_dur, 1e-9)
-    rust_mb_per_sec = (data_bytes / (1024 * 1024)) / max(rust_dur, 1e-9)
-
-    fallback_items_per_sec = workload_size / max(fallback_dur, 1e-9)
-    fallback_mb_per_sec = (data_bytes / (1024 * 1024)) / max(fallback_dur, 1e-9)
-
-    speedup = fallback_dur / max(rust_dur, 1e-9)
-
-    return BenchmarkResult(
-        benchmark_name=name,
-        workload_size=workload_size,
-        data_bytes=data_bytes,
-        rust_duration_sec=rust_dur,
-        rust_throughput_items_per_sec=rust_items_per_sec,
-        rust_throughput_mb_per_sec=rust_mb_per_sec,
-        fallback_duration_sec=fallback_dur,
-        fallback_throughput_items_per_sec=fallback_items_per_sec,
-        fallback_throughput_mb_per_sec=fallback_mb_per_sec,
-        speedup_ratio=speedup,
-    )
-
-
-def benchmark_text_cleaner(records: List[str], total_bytes: int) -> BenchmarkResult:
-    def rust_call():
-        return rs.clean_batch_records(records, True, True, 10, 10000, False, None)
-
-    def fallback_call():
-        return fb.clean_batch_records(records, True, True, 10, 10000, False, None)
-
-    return run_timed_benchmark(
-        name="TextCleaner.clean_batch (CRLF + Trimming + Bounds)",
-        rust_fn=rust_call,
-        fallback_fn=fallback_call,
-        workload_size=len(records),
-        data_bytes=total_bytes,
-    )
-
-
-def benchmark_quality_filter(records: List[str], total_bytes: int) -> BenchmarkResult:
-    rust_filter = rs.QualityFilter(
-        min_chars=10,
-        max_chars=20000,
-        min_words=3,
-        min_entropy=2.0,
-        max_repetition_ratio=0.3,
-        max_symbol_ratio=0.25,
-    )
-    fallback_filter = fb.QualityFilter(
-        min_chars=10,
-        max_chars=20000,
-        min_words=3,
-        min_entropy=2.0,
-        max_repetition_ratio=0.3,
-        max_symbol_ratio=0.25,
-    )
-
-    def rust_call():
-        return rust_filter.filter_batch(records)
-
-    def fallback_call():
-        return fallback_filter.filter_batch(records)
-
-    return run_timed_benchmark(
-        name="QualityFilter.filter_batch (Entropy + Repetition + Symbols)",
-        rust_fn=rust_call,
-        fallback_fn=fallback_call,
-        workload_size=len(records),
-        data_bytes=total_bytes,
-    )
-
-
-def benchmark_stream_dedup(records: List[str], total_bytes: int) -> BenchmarkResult:
-    # Inject 25% duplicate records
-    dup_records = list(records)
-    for i in range(len(dup_records) // 4):
-        dup_records[i * 4] = dup_records[i * 4 + 1]
-
-    def rust_call():
-        dedup = rs.StreamDeduplicator()
-        passed = dedup.filter_batch(dup_records)
-        dedup.commit()
-        return passed
-
-    def fallback_call():
-        dedup = fb.StreamDeduplicator()
-        passed = dedup.filter_batch(dup_records)
-        dedup.commit()
-        return passed
-
-    return run_timed_benchmark(
-        name="StreamDeduplicator.filter_batch (Exact XXH3-64 Staging & Commit)",
-        rust_fn=rust_call,
-        fallback_fn=fallback_call,
-        workload_size=len(dup_records),
-        data_bytes=total_bytes,
-    )
-
-
-def benchmark_minhash_dedup(records: List[str], total_bytes: int) -> BenchmarkResult:
-    # Use 500 documents for MinHash permutation benchmarking
-    sample_records = records[:500]
-    sample_bytes = sum(len(r.encode("utf-8")) for r in sample_records)
-
-    def rust_call():
-        dedup = rs.MinHashDeduplicator(0.8, 5, 64)
-        for r in sample_records:
-            dedup.is_unique(r)
-        return dedup.count()
-
-    def fallback_call():
-        dedup = fb.MinHashDeduplicator(0.8, 5, 64)
-        for r in sample_records:
-            dedup.is_unique(r)
-        return dedup.count()
-
-    return run_timed_benchmark(
-        name="MinHashDeduplicator.is_unique (64-Permutation MinHash Signatures)",
-        rust_fn=rust_call,
-        fallback_fn=fallback_call,
-        workload_size=len(sample_records),
-        data_bytes=sample_bytes,
-    )
-
-
-def benchmark_state_tracker() -> BenchmarkResult:
-    with tempfile.TemporaryDirectory() as tmpdir:
-        rust_wal = Path(tmpdir) / "rust_tracker.wal"
-        fb_wal = Path(tmpdir) / "fb_tracker.wal"
-
-        rust_tracker = rs.StateTracker(str(rust_wal))
-        fallback_tracker = fb.StateTracker(str(fb_wal))
-
-        num_commits = 2000
-
-        def rust_call():
-            for i in range(num_commits):
-                rust_tracker.update(1, 1, i * 32, i * 1024, 0)
-            rust_tracker.commit()
-
-        def fallback_call():
-            for i in range(num_commits):
-                fallback_tracker.update(1, 1, i * 32, i * 1024, 0)
-            fallback_tracker.commit()
-
-        return run_timed_benchmark(
-            name="StateTracker.update & commit (128-Byte Atomic CRC32 WAL)",
-            rust_fn=rust_call,
-            fallback_fn=fallback_call,
-            workload_size=num_commits,
-            data_bytes=num_commits * 128,
-        )
-
-
-def benchmark_string_tensor(records: List[str], total_bytes: int) -> BenchmarkResult:
-    def rust_call():
-        st = rs.StringTensor.from_list(records)
-        hashes = st.hash_all()
-        filtered = st.filter_by_length(20, 1000)
-        return len(filtered.to_list()), len(hashes)
-
-    def fallback_call():
-        st = fb.StringTensor.from_list(records)
-        hashes = st.hash_all()
-        filtered = st.filter_by_length(20, 1000)
-        return len(filtered.to_list()), len(hashes)
-
-    return run_timed_benchmark(
-        name="StringTensor (Umbra-style Layout, Hash All & Length Filter)",
-        rust_fn=rust_call,
-        fallback_fn=fallback_call,
-        workload_size=len(records),
-        data_bytes=total_bytes,
-    )
-
-
-def benchmark_fused_pipeline(records: List[str], total_bytes: int) -> BenchmarkResult:
-    def rust_call():
-        dedup = rs.StreamDeduplicator()
-        st = rs.fused_pipeline_process_batch(
-            records=records,
-            trim_whitespace=True,
-            normalize_newlines=True,
-            min_byte_len=10,
-            max_byte_len=10000,
-            truncate_over_max=False,
-            json_extract_key=None,
-            filter_quality=True,
-            min_chars=10,
-            max_chars=10000,
-            min_words=3,
-            min_entropy=2.0,
-            max_repetition_ratio=0.3,
-            max_symbol_ratio=0.25,
-            deduplicator=dedup,
-            commit_dedup=True,
-        )
-        return st.total_bytes()
-
-    def fallback_call():
-        dedup = fb.StreamDeduplicator()
-        st = fb.fused_pipeline_process_batch(
-            records=records,
-            trim_whitespace=True,
-            normalize_newlines=True,
-            min_byte_len=10,
-            max_byte_len=10000,
-            truncate_over_max=False,
-            json_extract_key=None,
-            filter_quality=True,
-            min_chars=10,
-            max_chars=10000,
-            min_words=3,
-            min_entropy=2.0,
-            max_repetition_ratio=0.3,
-            max_symbol_ratio=0.25,
-            deduplicator=dedup,
-            commit_dedup=True,
-        )
-        return st.total_bytes()
-
-    return run_timed_benchmark(
-        name="FusedPipeline.fused_process_batch (End-to-End SIMD Single Pass)",
-        rust_fn=rust_call,
-        fallback_fn=fallback_call,
-        workload_size=len(records),
-        data_bytes=total_bytes,
-    )
-
-
-def benchmark_serialization(records: List[str]) -> List[BenchmarkResult]:
-    cols = {
-        "id": [str(i) for i in range(len(records))],
-        "text": records,
-        "score": [str(round(math.sin(i) * 10, 4)) for i in range(len(records))],
-    }
-    total_data_bytes = sum(len(c.encode("utf-8")) for col in cols.values() for c in col)
-
-    results = []
-    for fmt in ["parquet", "arrow", "jsonl", "csv"]:
-        def rust_call(fmt=fmt):
-            return rs.serialize_table_to_bytes(cols, fmt, False)
-
-        def fallback_call(fmt=fmt):
-            return fb.serialize_table_to_bytes(cols, fmt, False)
-
-        res = run_timed_benchmark(
-            name=f"Serialization: {fmt.upper()}",
-            rust_fn=rust_call,
-            fallback_fn=fallback_call,
-            workload_size=len(records),
-            data_bytes=total_data_bytes,
-        )
-        results.append(res)
-    return results
-
-
-def main() -> None:
-    print("=================================================================")
-    print("  LuminaMoon Data Processing Engine Benchmark Suite")
-    print(f"  CPU Architecture / SIMD ISA: {rs.dataloader_cpu_isa()}")
-    print("=================================================================\n")
-
-    num_records = 10000
-    print(f"Generating synthetic workload ({num_records:,} records)...")
-    records, total_bytes = generate_synthetic_dataset(num_records=num_records)
-    print(f"Generated {len(records):,} records, {total_bytes / (1024*1024):.2f} MB payload.\n")
-
-    benchmarks = [
-        ("Text Cleaner", lambda: benchmark_text_cleaner(records, total_bytes)),
-        ("Quality Filter", lambda: benchmark_quality_filter(records, total_bytes)),
-        ("Stream Deduplicator", lambda: benchmark_stream_dedup(records, total_bytes)),
-        ("MinHash Deduplicator", lambda: benchmark_minhash_dedup(records, total_bytes)),
-        ("State Tracker", lambda: benchmark_state_tracker()),
-        ("String Tensor", lambda: benchmark_string_tensor(records, total_bytes)),
-        ("Fused Pipeline", lambda: benchmark_fused_pipeline(records, total_bytes)),
-    ]
-
-    all_results: List[BenchmarkResult] = []
-
-    for label, bench_fn in benchmarks:
-        print(f"Running benchmark: {label}...")
-        res = bench_fn()
-        all_results.append(res)
-        print(f"  -> Rust:     {res.rust_throughput_items_per_sec:>10,.0f} items/s ({res.rust_throughput_mb_per_sec:>8.2f} MB/s, {res.rust_duration_sec*1000:>7.2f} ms)")
-        print(f"  -> Fallback: {res.fallback_throughput_items_per_sec:>10,.0f} items/s ({res.fallback_throughput_mb_per_sec:>8.2f} MB/s, {res.fallback_duration_sec*1000:>7.2f} ms)")
-        print(f"  -> Speedup:  {res.speedup_ratio:>7.2f}x\n")
-
-    print("Running Serialization Benchmarks...")
-    ser_results = benchmark_serialization(records[:5000])
-    for res in ser_results:
-        all_results.append(res)
-        print(f"  -> {res.benchmark_name:<28}: Rust {res.rust_throughput_mb_per_sec:>8.2f} MB/s vs Fallback {res.fallback_throughput_mb_per_sec:>8.2f} MB/s (Speedup: {res.speedup_ratio:>6.2f}x)")
-
-    # Save output json
-    out_dir = Path(__file__).resolve().parent.parent / "logs"
-    out_dir.mkdir(parents=True, exist_ok=True)
-    out_file = out_dir / "benchmark_data_processing_results.json"
-    with open(out_file, "w", encoding="utf-8") as f:
-        json.dump(
-            {
-                "cpu_isa": rs.dataloader_cpu_isa(),
-                "num_records": num_records,
-                "total_mb": total_bytes / (1024 * 1024),
-                "benchmarks": [asdict(r) for r in all_results],
-            },
-            f,
-            indent=2,
-        )
-    print(f"\nSaved raw benchmark metrics to {out_file}")
-
-
-if __name__ == "__main__":
-    main()
+"""LuminaMoon Data Processing Engine Benchmark.
+
+Compares Native Rust SIMD Accelerator (AVX-512 / AVX2 / Rayon) against
+Pure Python Fallbacks across micro-benchmarks and end-to-end pipelines.
+"""
+
+from __future__ import annotations
+
+import json
+import math
+import os
+import random
+import sys
+import tempfile
+import time
+from dataclasses import asdict, dataclass
+from pathlib import Path
+from typing import Any, Callable, Dict, List, Tuple
+
+# Ensure local python package is discoverable
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "python"))
+
+from luminamoon.optimizer import _luminamoon_optimizer_rs as rs
+from luminamoon.data import fallbacks as fb
+
+
+@dataclass
+class BenchmarkResult:
+    benchmark_name: str
+    workload_size: int
+    data_bytes: int
+    rust_duration_sec: float
+    rust_throughput_items_per_sec: float
+    rust_throughput_mb_per_sec: float
+    fallback_duration_sec: float
+    fallback_throughput_items_per_sec: float
+    fallback_throughput_mb_per_sec: float
+    speedup_ratio: float
+
+
+def generate_synthetic_dataset(num_records: int = 10000, seed: int = 42) -> Tuple[List[str], int]:
+    rng = random.Random(seed)
+    templates = [
+        "   The quick brown fox jumps over the lazy dog.\r\n   This is line 2 with carriage returns.\r\n",
+        "Deep learning optimization requires efficient gradient descent with momentum and adaptive learning rates.\n",
+        '{"text": "{\"key\": \"value\", \"nested\": [1, 2, 3]}", "meta": "test record"}\r\n',
+        "System security alert: detected unauthorized access attempt from 192.168.1.105 on port 443.\n",
+        "Contact user support at alice.walker@enterprise-domain.org or call +1-800-555-0199 for assistance.\r\n",
+        "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n",  # high repetition
+        "Short text.\n",
+        "Mathematics: Pade [13/6] approximant ensures subnormal FP16 RNE preservation without overflow.\n",
+        "High symbol noise: #$%@!&*()!@#$%^&*()_+{}|:<>?~`!@#$%^&*()_+{}|:<>?~`\n",
+        "Large document block: " + ("lorem ipsum dolor sit amet " * 20) + "\r\n",
+    ]
+    records = []
+    total_bytes = 0
+    for i in range(num_records):
+        base = templates[i % len(templates)]
+        noise = f" [doc_id={i}_{rng.randint(100, 999)}]"
+        rec = base + noise
+        records.append(rec)
+        total_bytes += len(rec.encode("utf-8"))
+    return records, total_bytes
+
+
+def run_timed_benchmark(
+    name: str,
+    rust_fn: Callable[[], Any],
+    fallback_fn: Callable[[], Any],
+    workload_size: int,
+    data_bytes: int,
+    warmup_runs: int = 2,
+    test_runs: int = 5,
+) -> BenchmarkResult:
+    # Warmup Rust
+    for _ in range(warmup_runs):
+        rust_fn()
+    # Timed Rust
+    rust_times = []
+    for _ in range(test_runs):
+        t0 = time.perf_counter_ns()
+        rust_fn()
+        t1 = time.perf_counter_ns()
+        rust_times.append((t1 - t0) / 1e9)
+    rust_dur = min(rust_times)  # best of N runs
+
+    # Warmup Fallback
+    for _ in range(warmup_runs):
+        fallback_fn()
+    # Timed Fallback
+    fallback_times = []
+    for _ in range(test_runs):
+        t0 = time.perf_counter_ns()
+        fallback_fn()
+        t1 = time.perf_counter_ns()
+        fallback_times.append((t1 - t0) / 1e9)
+    fallback_dur = min(fallback_times)
+
+    rust_items_per_sec = workload_size / max(rust_dur, 1e-9)
+    rust_mb_per_sec = (data_bytes / (1024 * 1024)) / max(rust_dur, 1e-9)
+
+    fallback_items_per_sec = workload_size / max(fallback_dur, 1e-9)
+    fallback_mb_per_sec = (data_bytes / (1024 * 1024)) / max(fallback_dur, 1e-9)
+
+    speedup = fallback_dur / max(rust_dur, 1e-9)
+
+    return BenchmarkResult(
+        benchmark_name=name,
+        workload_size=workload_size,
+        data_bytes=data_bytes,
+        rust_duration_sec=rust_dur,
+        rust_throughput_items_per_sec=rust_items_per_sec,
+        rust_throughput_mb_per_sec=rust_mb_per_sec,
+        fallback_duration_sec=fallback_dur,
+        fallback_throughput_items_per_sec=fallback_items_per_sec,
+        fallback_throughput_mb_per_sec=fallback_mb_per_sec,
+        speedup_ratio=speedup,
+    )
+
+
+def benchmark_text_cleaner(records: List[str], total_bytes: int) -> BenchmarkResult:
+    def rust_call():
+        return rs.clean_batch_records(records, True, True, 10, 10000, False, None)
+
+    def fallback_call():
+        return fb.clean_batch_records(records, True, True, 10, 10000, False, None)
+
+    return run_timed_benchmark(
+        name="TextCleaner.clean_batch (CRLF + Trimming + Bounds)",
+        rust_fn=rust_call,
+        fallback_fn=fallback_call,
+        workload_size=len(records),
+        data_bytes=total_bytes,
+    )
+
+
+def benchmark_quality_filter(records: List[str], total_bytes: int) -> BenchmarkResult:
+    rust_filter = rs.QualityFilter(
+        min_chars=10,
+        max_chars=20000,
+        min_words=3,
+        min_entropy=2.0,
+        max_repetition_ratio=0.3,
+        max_symbol_ratio=0.25,
+    )
+    fallback_filter = fb.QualityFilter(
+        min_chars=10,
+        max_chars=20000,
+        min_words=3,
+        min_entropy=2.0,
+        max_repetition_ratio=0.3,
+        max_symbol_ratio=0.25,
+    )
+
+    def rust_call():
+        return rust_filter.filter_batch(records)
+
+    def fallback_call():
+        return fallback_filter.filter_batch(records)
+
+    return run_timed_benchmark(
+        name="QualityFilter.filter_batch (Entropy + Repetition + Symbols)",
+        rust_fn=rust_call,
+        fallback_fn=fallback_call,
+        workload_size=len(records),
+        data_bytes=total_bytes,
+    )
+
+
+def benchmark_stream_dedup(records: List[str], total_bytes: int) -> BenchmarkResult:
+    # Inject 25% duplicate records
+    dup_records = list(records)
+    for i in range(len(dup_records) // 4):
+        dup_records[i * 4] = dup_records[i * 4 + 1]
+
+    def rust_call():
+        dedup = rs.StreamDeduplicator()
+        passed = dedup.filter_batch(dup_records)
+        dedup.commit()
+        return passed
+
+    def fallback_call():
+        dedup = fb.StreamDeduplicator()
+        passed = dedup.filter_batch(dup_records)
+        dedup.commit()
+        return passed
+
+    return run_timed_benchmark(
+        name="StreamDeduplicator.filter_batch (Exact XXH3-64 Staging & Commit)",
+        rust_fn=rust_call,
+        fallback_fn=fallback_call,
+        workload_size=len(dup_records),
+        data_bytes=total_bytes,
+    )
+
+
+def benchmark_minhash_dedup(records: List[str], total_bytes: int) -> BenchmarkResult:
+    # Use 500 documents for MinHash permutation benchmarking
+    sample_records = records[:500]
+    sample_bytes = sum(len(r.encode("utf-8")) for r in sample_records)
+
+    def rust_call():
+        dedup = rs.MinHashDeduplicator(0.8, 5, 64)
+        for r in sample_records:
+            dedup.is_unique(r)
+        return dedup.count()
+
+    def fallback_call():
+        dedup = fb.MinHashDeduplicator(0.8, 5, 64)
+        for r in sample_records:
+            dedup.is_unique(r)
+        return dedup.count()
+
+    return run_timed_benchmark(
+        name="MinHashDeduplicator.is_unique (64-Permutation MinHash Signatures)",
+        rust_fn=rust_call,
+        fallback_fn=fallback_call,
+        workload_size=len(sample_records),
+        data_bytes=sample_bytes,
+    )
+
+
+def benchmark_state_tracker() -> BenchmarkResult:
+    with tempfile.TemporaryDirectory() as tmpdir:
+        rust_wal = Path(tmpdir) / "rust_tracker.wal"
+        fb_wal = Path(tmpdir) / "fb_tracker.wal"
+
+        rust_tracker = rs.StateTracker(str(rust_wal))
+        fallback_tracker = fb.StateTracker(str(fb_wal))
+
+        num_commits = 2000
+
+        def rust_call():
+            for i in range(num_commits):
+                rust_tracker.update(1, 1, i * 32, i * 1024, 0)
+            rust_tracker.commit()
+
+        def fallback_call():
+            for i in range(num_commits):
+                fallback_tracker.update(1, 1, i * 32, i * 1024, 0)
+            fallback_tracker.commit()
+
+        return run_timed_benchmark(
+            name="StateTracker.update & commit (128-Byte Atomic CRC32 WAL)",
+            rust_fn=rust_call,
+            fallback_fn=fallback_call,
+            workload_size=num_commits,
+            data_bytes=num_commits * 128,
+        )
+
+
+def benchmark_string_tensor(records: List[str], total_bytes: int) -> BenchmarkResult:
+    def rust_call():
+        st = rs.StringTensor.from_list(records)
+        hashes = st.hash_all()
+        filtered = st.filter_by_length(20, 1000)
+        return len(filtered.to_list()), len(hashes)
+
+    def fallback_call():
+        st = fb.StringTensor.from_list(records)
+        hashes = st.hash_all()
+        filtered = st.filter_by_length(20, 1000)
+        return len(filtered.to_list()), len(hashes)
+
+    return run_timed_benchmark(
+        name="StringTensor (Umbra-style Layout, Hash All & Length Filter)",
+        rust_fn=rust_call,
+        fallback_fn=fallback_call,
+        workload_size=len(records),
+        data_bytes=total_bytes,
+    )
+
+
+def benchmark_fused_pipeline(records: List[str], total_bytes: int) -> BenchmarkResult:
+    def rust_call():
+        dedup = rs.StreamDeduplicator()
+        st = rs.fused_pipeline_process_batch(
+            records=records,
+            trim_whitespace=True,
+            normalize_newlines=True,
+            min_byte_len=10,
+            max_byte_len=10000,
+            truncate_over_max=False,
+            json_extract_key=None,
+            filter_quality=True,
+            min_chars=10,
+            max_chars=10000,
+            min_words=3,
+            min_entropy=2.0,
+            max_repetition_ratio=0.3,
+            max_symbol_ratio=0.25,
+            deduplicator=dedup,
+            commit_dedup=True,
+        )
+        return st.total_bytes()
+
+    def fallback_call():
+        dedup = fb.StreamDeduplicator()
+        st = fb.fused_pipeline_process_batch(
+            records=records,
+            trim_whitespace=True,
+            normalize_newlines=True,
+            min_byte_len=10,
+            max_byte_len=10000,
+            truncate_over_max=False,
+            json_extract_key=None,
+            filter_quality=True,
+            min_chars=10,
+            max_chars=10000,
+            min_words=3,
+            min_entropy=2.0,
+            max_repetition_ratio=0.3,
+            max_symbol_ratio=0.25,
+            deduplicator=dedup,
+            commit_dedup=True,
+        )
+        return st.total_bytes()
+
+    return run_timed_benchmark(
+        name="FusedPipeline.fused_process_batch (End-to-End SIMD Single Pass)",
+        rust_fn=rust_call,
+        fallback_fn=fallback_call,
+        workload_size=len(records),
+        data_bytes=total_bytes,
+    )
+
+
+def benchmark_serialization(records: List[str]) -> List[BenchmarkResult]:
+    cols = {
+        "id": [str(i) for i in range(len(records))],
+        "text": records,
+        "score": [str(round(math.sin(i) * 10, 4)) for i in range(len(records))],
+    }
+    total_data_bytes = sum(len(c.encode("utf-8")) for col in cols.values() for c in col)
+
+    results = []
+    for fmt in ["parquet", "arrow", "jsonl", "csv"]:
+        def rust_call(fmt=fmt):
+            return rs.serialize_table_to_bytes(cols, fmt, False)
+
+        def fallback_call(fmt=fmt):
+            return fb.serialize_table_to_bytes(cols, fmt, False)
+
+        res = run_timed_benchmark(
+            name=f"Serialization: {fmt.upper()}",
+            rust_fn=rust_call,
+            fallback_fn=fallback_call,
+            workload_size=len(records),
+            data_bytes=total_data_bytes,
+        )
+        results.append(res)
+    return results
+
+
+def main() -> None:
+    print("=================================================================")
+    print("  LuminaMoon Data Processing Engine Benchmark Suite")
+    print(f"  CPU Architecture / SIMD ISA: {rs.dataloader_cpu_isa()}")
+    print("=================================================================\n")
+
+    num_records = 10000
+    print(f"Generating synthetic workload ({num_records:,} records)...")
+    records, total_bytes = generate_synthetic_dataset(num_records=num_records)
+    print(f"Generated {len(records):,} records, {total_bytes / (1024*1024):.2f} MB payload.\n")
+
+    benchmarks = [
+        ("Text Cleaner", lambda: benchmark_text_cleaner(records, total_bytes)),
+        ("Quality Filter", lambda: benchmark_quality_filter(records, total_bytes)),
+        ("Stream Deduplicator", lambda: benchmark_stream_dedup(records, total_bytes)),
+        ("MinHash Deduplicator", lambda: benchmark_minhash_dedup(records, total_bytes)),
+        ("State Tracker", lambda: benchmark_state_tracker()),
+        ("String Tensor", lambda: benchmark_string_tensor(records, total_bytes)),
+        ("Fused Pipeline", lambda: benchmark_fused_pipeline(records, total_bytes)),
+    ]
+
+    all_results: List[BenchmarkResult] = []
+
+    for label, bench_fn in benchmarks:
+        print(f"Running benchmark: {label}...")
+        res = bench_fn()
+        all_results.append(res)
+        print(f"  -> Rust:     {res.rust_throughput_items_per_sec:>10,.0f} items/s ({res.rust_throughput_mb_per_sec:>8.2f} MB/s, {res.rust_duration_sec*1000:>7.2f} ms)")
+        print(f"  -> Fallback: {res.fallback_throughput_items_per_sec:>10,.0f} items/s ({res.fallback_throughput_mb_per_sec:>8.2f} MB/s, {res.fallback_duration_sec*1000:>7.2f} ms)")
+        print(f"  -> Speedup:  {res.speedup_ratio:>7.2f}x\n")
+
+    print("Running Serialization Benchmarks...")
+    ser_results = benchmark_serialization(records[:5000])
+    for res in ser_results:
+        all_results.append(res)
+        print(f"  -> {res.benchmark_name:<28}: Rust {res.rust_throughput_mb_per_sec:>8.2f} MB/s vs Fallback {res.fallback_throughput_mb_per_sec:>8.2f} MB/s (Speedup: {res.speedup_ratio:>6.2f}x)")
+
+    # Save output json
+    out_dir = Path(__file__).resolve().parent.parent / "logs"
+    out_dir.mkdir(parents=True, exist_ok=True)
+    out_file = out_dir / "benchmark_data_processing_results.json"
+    with open(out_file, "w", encoding="utf-8") as f:
+        json.dump(
+            {
+                "cpu_isa": rs.dataloader_cpu_isa(),
+                "num_records": num_records,
+                "total_mb": total_bytes / (1024 * 1024),
+                "benchmarks": [asdict(r) for r in all_results],
+            },
+            f,
+            indent=2,
+        )
+    print(f"\nSaved raw benchmark metrics to {out_file}")
+
+
+if __name__ == "__main__":
+    main()

benchmarks/bench_luminav_vision_modes.py

@@ -1,398 +1,398 @@
-#!/usr/bin/env python3
-"""Benchmark and Comparative Suite for LuminaV across all execution and buffer modes.
-
-Tests training performance on highly complex, pathological synthetic image data
-(multi-octave fractal turbulence, non-stationary Moire/Chirp waves, chaotic manifolds)
-using a deep convolutional vision architecture (ResNet-style ConvNet with residual blocks).
-"""
-
-from __future__ import annotations
-
-import math
-import os
-import sys
-import time
-from typing import Any, Dict, List, Tuple
-
-import numpy as np
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-
-from luminamoon import LuminaV
-
-
-# 1. Pathological / Highly Complex Synthetic Vision Dataset Generation
-def generate_hard_synthetic_vision_dataset(
-    num_samples: int = 256,
-    height: int = 64,
-    width: int = 64,
-    num_classes: int = 8,
-    seed: int = 2026,
-) -> Tuple[torch.Tensor, torch.Tensor]:
-    """Generates an extremely difficult synthetic image dataset.
-
-    Features:
-    - Channel 0: Multi-octave Fractal Brownian Motion (fBm) with non-stationary phase shifts.
-    - Channel 1: High-frequency chirp & radial Moire vortex with cross-quadrature modulation.
-    - Channel 2: Chaotic double-well reaction-diffusion attractor pattern with high spatial curvature.
-    - Target: Non-linear topological invariant coupling cross-channel gradients with label ambiguity.
-    """
-    rng = np.random.RandomState(seed)
-
-    # Coordinate grids normalized to [-2, 2]
-    y_coords, x_coords = np.meshgrid(
-        np.linspace(-2.0, 2.0, height, dtype=np.float32),
-        np.linspace(-2.0, 2.0, width, dtype=np.float32),
-        indexing="ij",
-    )
-    r_sq = x_coords**2 + y_coords**2
-    theta = np.arctan2(y_coords, x_coords)
-
-    images = np.zeros((num_samples, 3, height, width), dtype=np.float32)
-    labels = np.zeros((num_samples,), dtype=np.int64)
-
-    for i in range(num_samples):
-        # Sample-specific chaotic random parameters
-        omega = rng.uniform(8.0, 18.0)
-        phi = rng.uniform(0.0, 2.0 * math.pi)
-        k_oct = rng.uniform(3.0, 7.0)
-
-        # Ch 0: Fractal multi-octave turbulence
-        ch0 = np.zeros((height, width), dtype=np.float32)
-        for octave in range(1, 5):
-            freq = 2.0**octave
-            amp = 1.0 / freq
-            angle = phi + octave * 0.785
-            proj = x_coords * math.cos(angle) + y_coords * math.sin(angle)
-            ch0 += amp * np.sin(freq * k_oct * proj + phi)
-        # Add high-frequency micro-checkerboard jitter
-        checker = ((np.arange(height)[:, None] + np.arange(width)[None, :]) % 2) * 2.0 - 1.0
-        ch0 += 0.25 * checker * np.cos(omega * r_sq)
-
-        # Ch 1: High-frequency Chirp / Moire radial vortex
-        ch1 = np.sin(omega * r_sq + 4.0 * theta + phi) * np.cos(k_oct * x_coords * y_coords)
-        # Cross-quadrature modulation
-        ch1 += 0.35 * np.cos(15.0 * (x_coords + y_coords) + phi)
-
-        # Ch 2: Chaotic double-well / non-linear attractor
-        poly = (x_coords**3 - 3.0 * x_coords * (y_coords**2)) * 0.5
-        ch2 = np.tanh(poly * 2.0 + np.cos(omega * y_coords))
-        ch2 += 0.3 * np.sin(20.0 * x_coords + phi) * np.sin(20.0 * y_coords)
-
-        # Combine channels and normalize
-        images[i, 0] = (ch0 - ch0.mean()) / (ch0.std() + 1e-6)
-        images[i, 1] = (ch1 - ch1.mean()) / (ch1.std() + 1e-6)
-        images[i, 2] = (ch2 - ch2.mean()) / (ch2.std() + 1e-6)
-
-        # Compute complex topological functional for target class:
-        # Cross-gradient winding energy + multi-channel nonlinear coupling
-        dx_ch1 = np.gradient(ch1, axis=1)
-        dy_ch2 = np.gradient(ch2, axis=0)
-        curl_cross = np.mean(ch0 * (dx_ch1 * dy_ch2))
-        vol = np.mean(ch0 * ch1 * ch2)
-        quad = np.mean(np.sin(r_sq * 5.0) * ch0)
-
-        # Map to discrete class [0..num_classes-1] via non-linear hash
-        hash_val = math.sin(curl_cross * 25.0 + vol * 15.0 + quad * 10.0 + phi)
-        class_idx = int(math.floor(((hash_val + 1.0) / 2.0) * num_classes)) % num_classes
-
-        # Inject 10% label ambiguity / boundary noise to test cautious clamping & regularization
-        if rng.rand() < 0.10:
-            class_idx = rng.randint(0, num_classes)
-
-        labels[i] = class_idx
-
-    return torch.from_numpy(images), torch.from_numpy(labels)
-
-
-# 2. Deep Convolutional Vision Architecture (ResNet-Lite)
-class ResidualConvBlock(nn.Module):
-    def __init__(self, channels: int):
-        super().__init__()
-        self.conv1 = nn.Conv2d(channels, channels, kernel_size=3, padding=1, bias=False)
-        self.bn1 = nn.BatchNorm2d(channels)
-        self.conv2 = nn.Conv2d(channels, channels, kernel_size=3, padding=1, bias=False)
-        self.bn2 = nn.BatchNorm2d(channels)
-
-    def forward(self, x: torch.Tensor) -> torch.Tensor:
-        res = x
-        out = F.silu(self.bn1(self.conv1(x)))
-        out = self.bn2(self.conv2(out))
-        return F.silu(out + res)
-
-
-class HardVisionConvNet(nn.Module):
-    def __init__(self, in_channels: int = 3, num_classes: int = 8):
-        super().__init__()
-        # Initial stem
-        self.stem = nn.Sequential(
-            nn.Conv2d(in_channels, 32, kernel_size=3, padding=1, bias=False),
-            nn.BatchNorm2d(32),
-            nn.SiLU(),
-        )
-        # Stage 1 (32 channels)
-        self.res1 = ResidualConvBlock(32)
-        self.pool1 = nn.MaxPool2d(2)  # 64 -> 32
-
-        # Stage 2 (64 channels)
-        self.trans1 = nn.Sequential(
-            nn.Conv2d(32, 64, kernel_size=3, padding=1, bias=False),
-            nn.BatchNorm2d(64),
-            nn.SiLU(),
-        )
-        self.res2 = ResidualConvBlock(64)
-        self.pool2 = nn.MaxPool2d(2)  # 32 -> 16
-
-        # Stage 3 (128 channels)
-        self.trans2 = nn.Sequential(
-            nn.Conv2d(64, 128, kernel_size=3, padding=1, bias=False),
-            nn.BatchNorm2d(128),
-            nn.SiLU(),
-        )
-        self.res3 = ResidualConvBlock(128)
-        self.pool3 = nn.AdaptiveAvgPool2d((2, 2))  # 128 x 2 x 2 = 512
-
-        # Classification Head
-        self.classifier = nn.Sequential(
-            nn.Linear(512, 128),
-            nn.SiLU(),
-            nn.Dropout(0.1),
-            nn.Linear(128, num_classes),
-        )
-
-    def forward(self, x: torch.Tensor) -> torch.Tensor:
-        x = self.stem(x)
-        x = self.res1(x)
-        x = self.pool1(x)
-        x = self.trans1(x)
-        x = self.res2(x)
-        x = self.pool2(x)
-        x = self.trans2(x)
-        x = self.res3(x)
-        x = self.pool3(x)
-        x = torch.flatten(x, 1)
-        return self.classifier(x)
-
-
-# 3. Benchmark Training Routine for Single Configuration
-def run_benchmark_trial(
-    config_name: str,
-    optimizer_fn: Any,
-    images: torch.Tensor,
-    labels: torch.Tensor,
-    batch_size: int = 64,
-    steps: int = 20,
-    seed: int = 42,
-) -> Dict[str, Any]:
-    torch.manual_seed(seed)
-    np.random.seed(seed)
-
-    model = HardVisionConvNet(in_channels=3, num_classes=8)
-    criterion = nn.CrossEntropyLoss()
-    optimizer = optimizer_fn(model.parameters())
-
-    total_params = sum(p.numel() for p in model.parameters())
-    num_tensors = sum(1 for p in model.parameters())
-
-    losses: List[float] = []
-    step_times: List[float] = []
-    accuracies: List[float] = []
-
-    num_samples = images.shape[0]
-
-    # Pre-split into deterministic batches
-    batch_indices = [
-        slice(
-            (s * batch_size) % (num_samples - batch_size),
-            ((s * batch_size) % (num_samples - batch_size)) + batch_size,
-        )
-        for s in range(steps)
-    ]
-
-    total_start = time.perf_counter()
-
-    for step_idx in range(steps):
-        b_slice = batch_indices[step_idx]
-        batch_x = images[b_slice]
-        batch_y = labels[b_slice]
-
-        t0 = time.perf_counter()
-
-        optimizer.zero_grad(set_to_none=True)
-        logits = model(batch_x)
-        loss = criterion(logits, batch_y)
-        loss.backward()
-        optimizer.step()
-
-        t1 = time.perf_counter()
-
-        loss_val = float(loss.item())
-        losses.append(loss_val)
-        step_times.append((t1 - t0) * 1000.0)  # ms
-
-        preds = torch.argmax(logits, dim=1)
-        acc = float((preds == batch_y).float().mean().item()) * 100.0
-        accuracies.append(acc)
-
-    total_duration_ms = (time.perf_counter() - total_start) * 1000.0
-
-    # Calculate optimizer state memory footprint
-    opt_state_bytes = 0
-    if hasattr(optimizer, "state"):
-        for p, st in optimizer.state.items():
-            if isinstance(st, dict):
-                for k, v in st.items():
-                    if isinstance(v, torch.Tensor):
-                        opt_state_bytes += v.numel() * v.element_size()
-                    elif isinstance(v, np.ndarray):
-                        opt_state_bytes += v.nbytes
-
-    initial_loss = losses[0]
-    final_loss = losses[-1]
-    pct_drop = ((initial_loss - final_loss) / initial_loss) * 100.0
-    avg_step_ms = float(np.mean(step_times))
-
-    return {
-        "name": config_name,
-        "initial_loss": initial_loss,
-        "final_loss": final_loss,
-        "loss_reduction_pct": pct_drop,
-        "final_accuracy": accuracies[-1],
-        "total_time_ms": total_duration_ms,
-        "avg_step_ms": avg_step_ms,
-        "state_mem_mb": opt_state_bytes / (1024.0 * 1024.0),
-        "total_params": total_params,
-        "num_tensors": num_tensors,
-        "losses": losses,
-    }
-
-
-# 4. Main Suite Comparing All Modes
-def main():
-    print("=" * 80)
-    print("LuminaV Vision Mode Benchmark & Comparative Training Suite")
-    print("=" * 80)
-
-    hw = LuminaV.detect_hardware()
-    print("[Hardware Environment]")
-    print(f"  CPU SIMD Vectorization: {hw.get('simd_backend')}")
-    print(f"  Vulkan GPU Acceleration: {hw.get('vulkan_available')} ({hw.get('gpu_device_name')})")
-    print(f"  Triton AOT Available:    {hw.get('triton_aot_available')} (arch: {hw.get('triton_arch')})")
-    print(f"  Wave32 Hardware Support: {hw.get('wave32_supported')}")
-    try:
-        import triton
-        print(f"  Triton Windows Package:  Version {triton.__version__} ({triton.__file__})")
-    except Exception as e:
-        print(f"  Triton Windows Package:  Failed ({e})")
-    print("-" * 80)
-
-    print("[Dataset Generation] Synthesizing pathological hard vision images (64x64, 3ch)...")
-    images, labels = generate_hard_synthetic_vision_dataset(
-        num_samples=256, height=64, width=64, num_classes=8, seed=2026
-    )
-    print(f"  Generated {images.shape[0]} images | Shape: {images.shape} | Labels: {labels.shape}")
-    print(f"  Label distribution: {torch.bincount(labels).tolist()}")
-    print("-" * 80)
-
-    # Define test configurations across all LuminaV modes and architectures
-    configurations = [
-        # --- Triton Windows Backends ---
-        (
-            "LuminaV (triton, Dual-Buffer)",
-            lambda p: LuminaV(p, lr=1e-3, buffer=2, execution="triton", bound=True, cautious=True),
-        ),
-        (
-            "LuminaV (triton, Single-Buffer TD-CLM)",
-            lambda p: LuminaV(p, lr=1e-3, buffer=1, execution="triton", bound=True, cautious=True),
-        ),
-        # --- Hardware Backends (Vulkan GPU, SIMD AVX-512, Scalar Core) ---
-        (
-            "LuminaV (vulkan, Dual-Buffer)",
-            lambda p: LuminaV(p, lr=1e-3, buffer=2, execution="vulkan", bound=True, cautious=True),
-        ),
-        (
-            "LuminaV (simd AVX-512, Dual-Buffer)",
-            lambda p: LuminaV(p, lr=1e-3, buffer=2, execution="simd", bound=True, cautious=True),
-        ),
-        (
-            "LuminaV (scalar, Dual-Buffer)",
-            lambda p: LuminaV(p, lr=1e-3, buffer=2, execution="scalar", bound=True, cautious=True),
-        ),
-        # --- Buffer Architectures ---
-        (
-            "LuminaV (auto, Single-Buffer TD-CLM)",
-            lambda p: LuminaV(p, lr=1e-3, buffer=1, execution="auto", bound=True, cautious=True),
-        ),
-        (
-            "LuminaV (auto, Dual-Buffer CLM)",
-            lambda p: LuminaV(p, lr=1e-3, buffer=2, execution="auto", bound=True, cautious=True),
-        ),
-        (
-            "LuminaV (auto, T1 Single-Buffer Triangle)",
-            lambda p: LuminaV(p, lr=1e-3, buffer="T1", execution="auto", bound=True, cautious=True),
-        ),
-        (
-            "LuminaV (auto, T2 Dual-Buffer Triangle)",
-            lambda p: LuminaV(p, lr=1e-3, buffer="T2", execution="auto", bound=True, cautious=True),
-        ),
-        # --- Advanced Vision Features ---
-        (
-            "LuminaV (auto, fuse_tensors=True)",
-            lambda p: LuminaV(p, lr=1e-3, buffer=2, execution="auto", fuse_tensors=True),
-        ),
-        (
-            "LuminaV (auto, radial bound)",
-            lambda p: LuminaV(p, lr=1e-3, buffer=2, execution="auto", bound=True, bound_type="radial"),
-        ),
-        # --- Baseline Comparison ---
-        (
-            "Baseline AdamW (PyTorch)",
-            lambda p: torch.optim.AdamW(p, lr=1e-3, betas=(0.9, 0.999), weight_decay=0.01),
-        ),
-    ]
-
-    results: List[Dict[str, Any]] = []
-
-    print("[Training Phase] Benchmarking 20 optimization steps per configuration on ResNet-Lite...")
-    for idx, (name, opt_fn) in enumerate(configurations, 1):
-        print(f"\n[{idx:02d}/{len(configurations):02d}] Running trial: {name} ...", flush=True)
-        res = run_benchmark_trial(
-            name, opt_fn, images, labels, batch_size=64, steps=20, seed=42
-        )
-        results.append(res)
-        print(
-            f"      Loss: {res['initial_loss']:.4f} -> {res['final_loss']:.4f} "
-            f"({res['loss_reduction_pct']:+.2f}%) | "
-            f"Acc: {res['final_accuracy']:.1f}% | "
-            f"Step: {res['avg_step_ms']:.2f} ms | "
-            f"State Mem: {res['state_mem_mb']:.2f} MB"
-        )
-
-    print("\n" + "=" * 105)
-    print("COMPREHENSIVE BENCHMARK COMPARISON TABLE")
-    print("=" * 105)
-    header = (
-        f"{'Mode Configuration':<40} | {'Init Loss':<10} | {'Final Loss':<10} | "
-        f"{'Reduction':<10} | {'Accuracy':<9} | {'Avg Step':<11} | {'State RAM':<9}"
-    )
-    print(header)
-    print("-" * 105)
-
-    for r in results:
-        row = (
-            f"{r['name']:<40} | "
-            f"{r['initial_loss']:<10.4f} | "
-            f"{r['final_loss']:<10.4f} | "
-            f"{r['loss_reduction_pct']:>8.2f}% | "
-            f"{r['final_accuracy']:>7.1f}% | "
-            f"{r['avg_step_ms']:>8.2f} ms | "
-            f"{r['state_mem_mb']:>7.2f} MB"
-        )
-        print(row)
-
-    print("=" * 105)
-
-
-if __name__ == "__main__":
-    main()
+#!/usr/bin/env python3
+"""Benchmark and Comparative Suite for LuminaV across all execution and buffer modes.
+
+Tests training performance on highly complex, pathological synthetic image data
+(multi-octave fractal turbulence, non-stationary Moire/Chirp waves, chaotic manifolds)
+using a deep convolutional vision architecture (ResNet-style ConvNet with residual blocks).
+"""
+
+from __future__ import annotations
+
+import math
+import os
+import sys
+import time
+from typing import Any, Dict, List, Tuple
+
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from luminamoon import LuminaV
+
+
+# 1. Pathological / Highly Complex Synthetic Vision Dataset Generation
+def generate_hard_synthetic_vision_dataset(
+    num_samples: int = 256,
+    height: int = 64,
+    width: int = 64,
+    num_classes: int = 8,
+    seed: int = 2026,
+) -> Tuple[torch.Tensor, torch.Tensor]:
+    """Generates an extremely difficult synthetic image dataset.
+
+    Features:
+    - Channel 0: Multi-octave Fractal Brownian Motion (fBm) with non-stationary phase shifts.
+    - Channel 1: High-frequency chirp & radial Moire vortex with cross-quadrature modulation.
+    - Channel 2: Chaotic double-well reaction-diffusion attractor pattern with high spatial curvature.
+    - Target: Non-linear topological invariant coupling cross-channel gradients with label ambiguity.
+    """
+    rng = np.random.RandomState(seed)
+
+    # Coordinate grids normalized to [-2, 2]
+    y_coords, x_coords = np.meshgrid(
+        np.linspace(-2.0, 2.0, height, dtype=np.float32),
+        np.linspace(-2.0, 2.0, width, dtype=np.float32),
+        indexing="ij",
+    )
+    r_sq = x_coords**2 + y_coords**2
+    theta = np.arctan2(y_coords, x_coords)
+
+    images = np.zeros((num_samples, 3, height, width), dtype=np.float32)
+    labels = np.zeros((num_samples,), dtype=np.int64)
+
+    for i in range(num_samples):
+        # Sample-specific chaotic random parameters
+        omega = rng.uniform(8.0, 18.0)
+        phi = rng.uniform(0.0, 2.0 * math.pi)
+        k_oct = rng.uniform(3.0, 7.0)
+
+        # Ch 0: Fractal multi-octave turbulence
+        ch0 = np.zeros((height, width), dtype=np.float32)
+        for octave in range(1, 5):
+            freq = 2.0**octave
+            amp = 1.0 / freq
+            angle = phi + octave * 0.785
+            proj = x_coords * math.cos(angle) + y_coords * math.sin(angle)
+            ch0 += amp * np.sin(freq * k_oct * proj + phi)
+        # Add high-frequency micro-checkerboard jitter
+        checker = ((np.arange(height)[:, None] + np.arange(width)[None, :]) % 2) * 2.0 - 1.0
+        ch0 += 0.25 * checker * np.cos(omega * r_sq)
+
+        # Ch 1: High-frequency Chirp / Moire radial vortex
+        ch1 = np.sin(omega * r_sq + 4.0 * theta + phi) * np.cos(k_oct * x_coords * y_coords)
+        # Cross-quadrature modulation
+        ch1 += 0.35 * np.cos(15.0 * (x_coords + y_coords) + phi)
+
+        # Ch 2: Chaotic double-well / non-linear attractor
+        poly = (x_coords**3 - 3.0 * x_coords * (y_coords**2)) * 0.5
+        ch2 = np.tanh(poly * 2.0 + np.cos(omega * y_coords))
+        ch2 += 0.3 * np.sin(20.0 * x_coords + phi) * np.sin(20.0 * y_coords)
+
+        # Combine channels and normalize
+        images[i, 0] = (ch0 - ch0.mean()) / (ch0.std() + 1e-6)
+        images[i, 1] = (ch1 - ch1.mean()) / (ch1.std() + 1e-6)
+        images[i, 2] = (ch2 - ch2.mean()) / (ch2.std() + 1e-6)
+
+        # Compute complex topological functional for target class:
+        # Cross-gradient winding energy + multi-channel nonlinear coupling
+        dx_ch1 = np.gradient(ch1, axis=1)
+        dy_ch2 = np.gradient(ch2, axis=0)
+        curl_cross = np.mean(ch0 * (dx_ch1 * dy_ch2))
+        vol = np.mean(ch0 * ch1 * ch2)
+        quad = np.mean(np.sin(r_sq * 5.0) * ch0)
+
+        # Map to discrete class [0..num_classes-1] via non-linear hash
+        hash_val = math.sin(curl_cross * 25.0 + vol * 15.0 + quad * 10.0 + phi)
+        class_idx = int(math.floor(((hash_val + 1.0) / 2.0) * num_classes)) % num_classes
+
+        # Inject 10% label ambiguity / boundary noise to test cautious clamping & regularization
+        if rng.rand() < 0.10:
+            class_idx = rng.randint(0, num_classes)
+
+        labels[i] = class_idx
+
+    return torch.from_numpy(images), torch.from_numpy(labels)
+
+
+# 2. Deep Convolutional Vision Architecture (ResNet-Lite)
+class ResidualConvBlock(nn.Module):
+    def __init__(self, channels: int):
+        super().__init__()
+        self.conv1 = nn.Conv2d(channels, channels, kernel_size=3, padding=1, bias=False)
+        self.bn1 = nn.BatchNorm2d(channels)
+        self.conv2 = nn.Conv2d(channels, channels, kernel_size=3, padding=1, bias=False)
+        self.bn2 = nn.BatchNorm2d(channels)
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        res = x
+        out = F.silu(self.bn1(self.conv1(x)))
+        out = self.bn2(self.conv2(out))
+        return F.silu(out + res)
+
+
+class HardVisionConvNet(nn.Module):
+    def __init__(self, in_channels: int = 3, num_classes: int = 8):
+        super().__init__()
+        # Initial stem
+        self.stem = nn.Sequential(
+            nn.Conv2d(in_channels, 32, kernel_size=3, padding=1, bias=False),
+            nn.BatchNorm2d(32),
+            nn.SiLU(),
+        )
+        # Stage 1 (32 channels)
+        self.res1 = ResidualConvBlock(32)
+        self.pool1 = nn.MaxPool2d(2)  # 64 -> 32
+
+        # Stage 2 (64 channels)
+        self.trans1 = nn.Sequential(
+            nn.Conv2d(32, 64, kernel_size=3, padding=1, bias=False),
+            nn.BatchNorm2d(64),
+            nn.SiLU(),
+        )
+        self.res2 = ResidualConvBlock(64)
+        self.pool2 = nn.MaxPool2d(2)  # 32 -> 16
+
+        # Stage 3 (128 channels)
+        self.trans2 = nn.Sequential(
+            nn.Conv2d(64, 128, kernel_size=3, padding=1, bias=False),
+            nn.BatchNorm2d(128),
+            nn.SiLU(),
+        )
+        self.res3 = ResidualConvBlock(128)
+        self.pool3 = nn.AdaptiveAvgPool2d((2, 2))  # 128 x 2 x 2 = 512
+
+        # Classification Head
+        self.classifier = nn.Sequential(
+            nn.Linear(512, 128),
+            nn.SiLU(),
+            nn.Dropout(0.1),
+            nn.Linear(128, num_classes),
+        )
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        x = self.stem(x)
+        x = self.res1(x)
+        x = self.pool1(x)
+        x = self.trans1(x)
+        x = self.res2(x)
+        x = self.pool2(x)
+        x = self.trans2(x)
+        x = self.res3(x)
+        x = self.pool3(x)
+        x = torch.flatten(x, 1)
+        return self.classifier(x)
+
+
+# 3. Benchmark Training Routine for Single Configuration
+def run_benchmark_trial(
+    config_name: str,
+    optimizer_fn: Any,
+    images: torch.Tensor,
+    labels: torch.Tensor,
+    batch_size: int = 64,
+    steps: int = 20,
+    seed: int = 42,
+) -> Dict[str, Any]:
+    torch.manual_seed(seed)
+    np.random.seed(seed)
+
+    model = HardVisionConvNet(in_channels=3, num_classes=8)
+    criterion = nn.CrossEntropyLoss()
+    optimizer = optimizer_fn(model.parameters())
+
+    total_params = sum(p.numel() for p in model.parameters())
+    num_tensors = sum(1 for p in model.parameters())
+
+    losses: List[float] = []
+    step_times: List[float] = []
+    accuracies: List[float] = []
+
+    num_samples = images.shape[0]
+
+    # Pre-split into deterministic batches
+    batch_indices = [
+        slice(
+            (s * batch_size) % (num_samples - batch_size),
+            ((s * batch_size) % (num_samples - batch_size)) + batch_size,
+        )
+        for s in range(steps)
+    ]
+
+    total_start = time.perf_counter()
+
+    for step_idx in range(steps):
+        b_slice = batch_indices[step_idx]
+        batch_x = images[b_slice]
+        batch_y = labels[b_slice]
+
+        t0 = time.perf_counter()
+
+        optimizer.zero_grad(set_to_none=True)
+        logits = model(batch_x)
+        loss = criterion(logits, batch_y)
+        loss.backward()
+        optimizer.step()
+
+        t1 = time.perf_counter()
+
+        loss_val = float(loss.item())
+        losses.append(loss_val)
+        step_times.append((t1 - t0) * 1000.0)  # ms
+
+        preds = torch.argmax(logits, dim=1)
+        acc = float((preds == batch_y).float().mean().item()) * 100.0
+        accuracies.append(acc)
+
+    total_duration_ms = (time.perf_counter() - total_start) * 1000.0
+
+    # Calculate optimizer state memory footprint
+    opt_state_bytes = 0
+    if hasattr(optimizer, "state"):
+        for p, st in optimizer.state.items():
+            if isinstance(st, dict):
+                for k, v in st.items():
+                    if isinstance(v, torch.Tensor):
+                        opt_state_bytes += v.numel() * v.element_size()
+                    elif isinstance(v, np.ndarray):
+                        opt_state_bytes += v.nbytes
+
+    initial_loss = losses[0]
+    final_loss = losses[-1]
+    pct_drop = ((initial_loss - final_loss) / initial_loss) * 100.0
+    avg_step_ms = float(np.mean(step_times))
+
+    return {
+        "name": config_name,
+        "initial_loss": initial_loss,
+        "final_loss": final_loss,
+        "loss_reduction_pct": pct_drop,
+        "final_accuracy": accuracies[-1],
+        "total_time_ms": total_duration_ms,
+        "avg_step_ms": avg_step_ms,
+        "state_mem_mb": opt_state_bytes / (1024.0 * 1024.0),
+        "total_params": total_params,
+        "num_tensors": num_tensors,
+        "losses": losses,
+    }
+
+
+# 4. Main Suite Comparing All Modes
+def main():
+    print("=" * 80)
+    print("LuminaV Vision Mode Benchmark & Comparative Training Suite")
+    print("=" * 80)
+
+    hw = LuminaV.detect_hardware()
+    print("[Hardware Environment]")
+    print(f"  CPU SIMD Vectorization: {hw.get('simd_backend')}")
+    print(f"  Vulkan GPU Acceleration: {hw.get('vulkan_available')} ({hw.get('gpu_device_name')})")
+    print(f"  Triton AOT Available:    {hw.get('triton_aot_available')} (arch: {hw.get('triton_arch')})")
+    print(f"  Wave32 Hardware Support: {hw.get('wave32_supported')}")
+    try:
+        import triton
+        print(f"  Triton Windows Package:  Version {triton.__version__} ({triton.__file__})")
+    except Exception as e:
+        print(f"  Triton Windows Package:  Failed ({e})")
+    print("-" * 80)
+
+    print("[Dataset Generation] Synthesizing pathological hard vision images (64x64, 3ch)...")
+    images, labels = generate_hard_synthetic_vision_dataset(
+        num_samples=256, height=64, width=64, num_classes=8, seed=2026
+    )
+    print(f"  Generated {images.shape[0]} images | Shape: {images.shape} | Labels: {labels.shape}")
+    print(f"  Label distribution: {torch.bincount(labels).tolist()}")
+    print("-" * 80)
+
+    # Define test configurations across all LuminaV modes and architectures
+    configurations = [
+        # --- Triton Windows Backends ---
+        (
+            "LuminaV (triton, Dual-Buffer)",
+            lambda p: LuminaV(p, lr=1e-3, buffer=2, execution="triton", bound=True, cautious=True),
+        ),
+        (
+            "LuminaV (triton, Single-Buffer TD-CLM)",
+            lambda p: LuminaV(p, lr=1e-3, buffer=1, execution="triton", bound=True, cautious=True),
+        ),
+        # --- Hardware Backends (Vulkan GPU, SIMD AVX-512, Scalar Core) ---
+        (
+            "LuminaV (vulkan, Dual-Buffer)",
+            lambda p: LuminaV(p, lr=1e-3, buffer=2, execution="vulkan", bound=True, cautious=True),
+        ),
+        (
+            "LuminaV (simd AVX-512, Dual-Buffer)",
+            lambda p: LuminaV(p, lr=1e-3, buffer=2, execution="simd", bound=True, cautious=True),
+        ),
+        (
+            "LuminaV (scalar, Dual-Buffer)",
+            lambda p: LuminaV(p, lr=1e-3, buffer=2, execution="scalar", bound=True, cautious=True),
+        ),
+        # --- Buffer Architectures ---
+        (
+            "LuminaV (auto, Single-Buffer TD-CLM)",
+            lambda p: LuminaV(p, lr=1e-3, buffer=1, execution="auto", bound=True, cautious=True),
+        ),
+        (
+            "LuminaV (auto, Dual-Buffer CLM)",
+            lambda p: LuminaV(p, lr=1e-3, buffer=2, execution="auto", bound=True, cautious=True),
+        ),
+        (
+            "LuminaV (auto, T1 Single-Buffer Triangle)",
+            lambda p: LuminaV(p, lr=1e-3, buffer="T1", execution="auto", bound=True, cautious=True),
+        ),
+        (
+            "LuminaV (auto, T2 Dual-Buffer Triangle)",
+            lambda p: LuminaV(p, lr=1e-3, buffer="T2", execution="auto", bound=True, cautious=True),
+        ),
+        # --- Advanced Vision Features ---
+        (
+            "LuminaV (auto, fuse_tensors=True)",
+            lambda p: LuminaV(p, lr=1e-3, buffer=2, execution="auto", fuse_tensors=True),
+        ),
+        (
+            "LuminaV (auto, radial bound)",
+            lambda p: LuminaV(p, lr=1e-3, buffer=2, execution="auto", bound=True, bound_type="radial"),
+        ),
+        # --- Baseline Comparison ---
+        (
+            "Baseline AdamW (PyTorch)",
+            lambda p: torch.optim.AdamW(p, lr=1e-3, betas=(0.9, 0.999), weight_decay=0.01),
+        ),
+    ]
+
+    results: List[Dict[str, Any]] = []
+
+    print("[Training Phase] Benchmarking 20 optimization steps per configuration on ResNet-Lite...")
+    for idx, (name, opt_fn) in enumerate(configurations, 1):
+        print(f"\n[{idx:02d}/{len(configurations):02d}] Running trial: {name} ...", flush=True)
+        res = run_benchmark_trial(
+            name, opt_fn, images, labels, batch_size=64, steps=20, seed=42
+        )
+        results.append(res)
+        print(
+            f"      Loss: {res['initial_loss']:.4f} -> {res['final_loss']:.4f} "
+            f"({res['loss_reduction_pct']:+.2f}%) | "
+            f"Acc: {res['final_accuracy']:.1f}% | "
+            f"Step: {res['avg_step_ms']:.2f} ms | "
+            f"State Mem: {res['state_mem_mb']:.2f} MB"
+        )
+
+    print("\n" + "=" * 105)
+    print("COMPREHENSIVE BENCHMARK COMPARISON TABLE")
+    print("=" * 105)
+    header = (
+        f"{'Mode Configuration':<40} | {'Init Loss':<10} | {'Final Loss':<10} | "
+        f"{'Reduction':<10} | {'Accuracy':<9} | {'Avg Step':<11} | {'State RAM':<9}"
+    )
+    print(header)
+    print("-" * 105)
+
+    for r in results:
+        row = (
+            f"{r['name']:<40} | "
+            f"{r['initial_loss']:<10.4f} | "
+            f"{r['final_loss']:<10.4f} | "
+            f"{r['loss_reduction_pct']:>8.2f}% | "
+            f"{r['final_accuracy']:>7.1f}% | "
+            f"{r['avg_step_ms']:>8.2f} ms | "
+            f"{r['state_mem_mb']:>7.2f} MB"
+        )
+        print(row)
+
+    print("=" * 105)
+
+
+if __name__ == "__main__":
+    main()

changelogs/LUMINAV_ALGORITHM_CHANGELOG.md

@@ -1,430 +1,430 @@
-# LuminaV Optimizer Algorithm Changelog
-
-All notable changes to the LuminaV Optimizer algorithm are documented in this file.
-
-The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
-
-* [Software Package Changelog (`luminamoon`)](SOFTWARE_CHANGELOG.md)
-
----
-
-## [1.6.0] - 2026-11-03
-
-### Added
-- **T-Family (Triangle Family) tanh-free Modes (`buffer="T1"` & `buffer="T2"`)**:
-  * **`buffer="T2"` (2-Buffer Triangle Family Mode)**: Keeps the 2-buffer state (`exp_avg` and `exp_avg_sq`) without using $\tanh$, computing the gradient-momentum residual $d_t = g_t - m _ {t-1}$ and bounding the lookahead update $\tilde{m}_t$ via the **Triangle Inequality Envelope** $\mathcal{E}_t^{(\text{T2})}$:
-
-    $\displaystyle d_t = g_t - m _ {t-1}, \qquad v_t = \beta_2 v _ {t-1} + (1 - \beta_2) d_t^2$
-
-    $\displaystyle \tilde{m} _ t = w_1 m _ {t-1} + w_2 g_t, \qquad w_1 = \beta_1^2, \qquad w_2 = 1 - \beta_1^2$
-
-    $\displaystyle \mathcal{E} _ t^{(\text{T2})} = w_1 |m _ {t-1}| + w_2 |g_t| + 0.5\lambda |d_t| + 0.5\lambda \sigma_t + \epsilon _ {\text{eff}}, \qquad u_t^{(\text{T2})} = \frac{\tilde{m} _ t}{\mathcal{E} _ t^{(\text{T2})}} \in (-1, 1)$
-
-    By the Triangle Inequality ($|w_1 m _ {t-1} + w_2 g_t| \le w_1 |m _ {t-1}| + w_2 |g_t| \le \mathcal{E} _ t^{(\text{T2})}$), $|u_t^{(\text{T2})}| < 1$ is guaranteed without evaluating $\tanh$. The term $0.5\lambda |d_t|$ acts as a **triangle-defect penalty**, and $0.5\lambda \sigma_t$ provides belief-variance damping.
-  * **`buffer="T1"` (1-Buffer Triangle Family Mode)**: Combines the single-buffer **Triangle-Defect** denominator $\mathcal{E} _ t^{(\text{T1})} = w_1 |m _ {t-1}| + w_2 |g_t| + \lambda |d_t| + \epsilon _ {\text{eff}}$ (using residual $d_t = g_t - m _ {t-1}$) with the rational update bounds below, reducing optimizer state memory by 50% via the Triangle Inequality.
-  * **Rational Update Bounds (for `T1` & `T2`)**: Replaces $\tanh$ and $\sqrt{\cdot}$ in the coordinate, radial, and FP16 update bounds with simple rational formulas:
-
-    $\displaystyle b_i = \mathrm{bound} _ {\text{ratio}} \cdot (|w_i| + 10^{-2}), \qquad \Delta w_i = \frac{b_i (b_i + |s_i|) s_i}{b_i (b_i + |s_i|) + s_i^2}, \qquad \mathrm{scale} _ {\text{radial}} = \frac{1 + r}{1 + r + r^2}$
-  * Keeps 100% backward compatibility and identical outputs for existing modes (`buffer=1`, `buffer=2`, and `version="1.5.5"`).
-
----
-
-## [1.5.5] - 2026-10-29
-
-### Added
-- **Contiguous Bucket Fusion (`fuse_tensors: bool = False`)**:
-  * Added opt-in contiguous parameter and state coalescing (`_build_contiguous_buckets`) that groups parameters within each `(param_group, device, dtype)` domain $\mathcal{B}_{g,d,\tau}$ into contiguous 1D storage (`flat_p`, `flat_g`, `flat_m`, `flat_v`, `flat_mp`) and rebinds `p.data`, `p.grad`, and `self.state[p]` entries as zero-copy views (`flat_*[offset : offset + n].view(p.shape)`).
-  * Eliminates per-tensor CPU-to-GPU kernel launch latency on fragmented vision and edge architectures by executing an entire bucket in a single fused kernel dispatch while computing `cautious_scale` and `radial` trust bounds across the bucket domain $\mathcal{B}_{g,d,\tau}$.
-  * Added in-place `zero_grad()` view preservation (`flat_g.zero_()`) and full `state_dict()` / `load_state_dict()` checkpoint compatibility without leaking temporary buffer keys (LSO-Eval Stage II compliant).
-- **Triton JIT Pre-Warmup (`prewarm_jit: bool = True`)**:
-  * Added automatic initialization-time Triton JIT compilation (`_prewarm_all_configs`) using 64-element dummy buffers (`128–256 bytes` VRAM) with `lr=0.0` and `weight_decay=0.0` across both Step 1 (`pass1` + `pass2`) and Step 2+ (`fused_single_pass` + `post_step_ema`) paths with target `BLOCK_SIZE` specialization (`1024` or `2048`), eliminating first-step compilation stalls.
-- **Bounded Bucket Chunking & Sequential In-Place Migration (`bucket_cap_mb: float = 500.0`)**:
-  * Bound contiguous bucket coalescing by an upper memory capacity ceiling $C_{\text{cap}} = 500\text{ MB}$, partitioning model parameters into ordered contiguous chunks $\mathcal{B}^{(1)}, \mathcal{B}^{(2)}, \dots, \mathcal{B}^{(K)}$ such that $\sum_{p \in \mathcal{B}^{(k)}} \text{bytes}(p) \le C_{\text{cap}}$ (with giant single layers $> C_{\text{cap}}$ isolated into individual buckets).
-  * Enforced sequential in-place migration with immediate parameter storage recycling (`del old_data`, `del old_grad`) and garbage collection, constraining peak transient memory allocation spikes during optimizer initialization to at most $\min(\text{model bytes}, C _ {\text{cap}})$.
-
----
-
-## [1.5.4] - 2026-10-29
-
-### Added
-- **In-Container Native Bitwise-BF16 State Promotion (`dtype_mode == 2`, `master_weights="none"`)**:
-  * Promoted 16-bit optimizer state buffers ($m_t$ via stochastic rounding `f32_to_bf16_bits_sr` and $v_t$ via round-to-nearest-even `f32_to_bf16_bits_rne`) to native Bitwise-BF16 stored inside the existing 16-bit container during pure `FP16` execution, expanding state exponent dynamic range to 8 bits ($1.18 \times 10^{-38}$) with zero additional VRAM overhead.
-
-### Changed
-- **Unified Denominator Stability Constant ($\epsilon_{\text{eff}} = \epsilon = 10^{-8}$)**:
-  * Unified the effective denominator constant $\epsilon_{\text{eff}} = \epsilon = 10^{-8}$ across all precision modes and hardware backends (replacing the legacy `FP16` floor clamp $\max(\epsilon, 10^{-4})$ introduced in `1.1.5`), preventing update velocity attenuation and late-stage convergence loss resets when $\sqrt{v_t} \ll 10^{-4}$.
-
-### Removed
-- **Centered Defect Compensation (`centered_defect`, $\mu_d$)**:
-  * Removed the `centered_defect` hyperparameter and the global defect mean reduction $\mu_d = \frac{1}{N}\sum_{i=1}^N (g_{t,i} - m_{t-1,i})$ across both Dual-Buffer Convex Lookahead Momentum (`buffer=2` / CLM) and Single-Buffer Triangle-Defect Convex Lookahead Momentum (`buffer=1` / TD-CLM).
-  * Simplified the instantaneous gradient-momentum residual to purely local coordinate-wise differences:
-
-    $\displaystyle d_t^{(\text{B1})} = g_t - m_{t-1}, \qquad d_t^{(\text{B2})} = g_t - m_t$
-
-  * Eliminated the pre-step global reduction pass (`_lumina_diff_sum_kernel` in Triton JIT/AOT, `diff_sum.comp` in Vulkan SPIR-V, `reduce_diff_sum_simd` in x86\_64 AVX-512/AVX2, and slot 5 of the 6-element `ScratchBuffer`), removing cross-lane/cross-rank reduction latency while retaining the 6-element physical `ScratchBuffer` layout (`[mask_sum, u_sq_sum, p_sq_sum, cautious_scale, bound_scale, _reserved]`, 5 active slots + 1 reserved slot for ABI alignment).
-
----
-
-## [1.5.3] - 2026-10-20
-
-No versions were published between 1.3.3 and 1.5.3. This release notes all changes made since version 1.3.3.
-
-### Added
-- **Single-Buffer Triangle-Defect Convex Lookahead Momentum (TD-CLM, `buffer=1`)**:
-  * Replaced the legacy 3-pass RMS single-buffer update with a single-pass formulation:
-
-    $\displaystyle \tilde{m} _ t = w_1 m _ {t-1} + w_2 g_t, \quad w_1 = \beta_1^2, \quad w_2 = 1 - \beta_1^2$
-
-    $\displaystyle \mathcal{E} _ t = w_1 |m _ {t-1}| + w_2 |g_t| + \lambda |d_t^{(\text{B1})}| + \epsilon _ {\text{eff}}$
-
-    $\displaystyle u_t^{(\text{B1})} = \frac{\tilde{m} _ t}{\mathcal{E} _ t} \in (-1, 1)$
-
-  * In 1.3.3, single-buffer mode required three sequential GPU kernel launches (`_lumina_v1_pass1_kernel`, `_lumina_v1_pass2_kernel`, `_lumina_v1_pass3_update_kernel`) and global atomic reductions across thread blocks to compute tensor-wide root-mean-square ($\text{rms} = \sqrt{\frac{1}{N}\sum \tilde{m} _ i^2 + \epsilon}$).
-  * TD-CLM replaces the tensor-wide reduction with coordinate-wise envelope bounding via the Triangle Inequality Envelope $\mathcal{E} _ t$. This reduces single-buffer execution to a single GPU pass and eliminates all $\tanh$ evaluations in the momentum path.
-  * Calibrated the defect penalty weight $\lambda$ to the symmetric spike-reversal equilibrium:
-
-    $\displaystyle \lambda^* = \frac{2(1 - \beta_1^2)^2}{4\beta_1^2 - 3}$
-
-    For the default $\beta_1 = 0.9$, $\lambda^* \approx 0.3008$, setting the default parameter to `lam=0.3`.
-
-- **Padé [13/6] Rational Approximation (`tanh_pade_f32`)**:
-  * Implemented a rational Padé [13/6] polynomial for hyperbolic tangent calculations:
-
-    $\displaystyle \tanh _ {\text{Pade}}(x) = \frac{x(1 + c_1 x^2 + c_2 x^4 + c_3 x^6 + c_4 x^8 + c_5 x^{10} + c_6 x^{12})}{1 + d_1 x^2 + d_2 x^4 + d_3 x^6}$
-
-  * Evaluated over the clamped domain $x \in [-7.9988117, 7.9988117]$ with asymptotic saturation to $\pm 1.0$.
-  * In 1.3.3, `_triton_tanh_fast` used hardware-dependent SFU instructions (`tl.math.tanh`) or analytical sigmoid fallbacks (`2 / (1 + exp(-2x)) - 1`). Different GPU architectures (e.g. Turing, Ampere, Hopper) evaluate SFU transcendental functions with slightly different mantissa rounding, leading to bitwise divergence during distributed training.
-  * Padé [13/6] provides uniform precision with $\lVert \tanh _ {\text{Pade}}(x) - \tanh(x) \rVert _ \infty < 1.5 \times 10^{-6}$ across CPU Scalar, AVX2, AVX-512, Vulkan SPIR-V, and Triton GPU kernels.
-
-- **Centered Innovation Defect (`centered_defect`)**:
-  * Added batch-mean centering for gradient innovation tracking (`centered_defect=False` by default):
-
-    $\displaystyle \mu_d = \frac{1}{N}\sum _ {i=1}^N (g _ {t,i} - m _ {t-1,i})$
-
-    $\displaystyle d_t^{(\text{B1})} = (g_t - m _ {t-1}) - \mu_d$
-
-    $\displaystyle d_t^{(\text{B2})} = (g_t - m_t) - \beta_1 \mu_d$
-
-  * In 1.3.3, uncentered innovation $g_t - m_t$ was used directly. On layers where gradients have a non-zero mean (such as normalization scales, biases, and token embeddings), the raw difference caused systematic drift in the second-moment accumulator. Centering eliminates this mean shift.
-
-- **Subnormal-Safe IEEE-754 FP16 Stochastic Rounding**:
-  * Added `f32_to_f16_bits_rne` to handle IEEE-754 subnormal numbers ($|x| < 2^{-14}$) during FP16 stochastic rounding.
-  * In 1.3.3, stochastic rounding added 13-bit random noise and masked with `& -8192`. When a floating-point value entered the subnormal range, bitwise addition directly corrupted the exponent field (bit 10), causing values to flip between normal and subnormal representations or trigger hardware flush-to-zero traps.
-  * The updated routine checks for zero exponent, shifts the significand into the subnormal format, and applies round-to-nearest-even tie-breaking before bit truncation.
-
-- **Multi-Backend Execution Engine**:
-  * Replaced the Python-only runtime with a multi-backend execution layer:
-    * **Rust SIMD (`luminav-simd`)**: AVX-512 (512-bit registers), AVX2 with FMA, portable scalar fallback, and Rayon parallel chunk scheduling (`ELEM_CHUNK_SIZE = 2,097,152`).
-    * **Vulkan UMA (`luminav-vulkan`)**: SPIR-V compute pipeline for cross-vendor GPUs (AMD, Intel, Apple Silicon via MoltenVK, and NVIDIA) with zero-copy host-visible memory buffers.
-    * **Triton AOT (`luminav-triton-aot`)**: C-ABI wrapper dispatching ahead-of-time precompiled kernels for 84 GPU architectures (Blackwell SM100/SM120, Hopper SM90, Ada SM89, Ampere SM80/SM86, Turing SM75, and AMD ROCm gfx90a/gfx942/gfx1100).
-    * **Standalone Runtime (`luminamoon.optimizer.standalone`)**: Pure NumPy and C-ABI execution that runs without requiring PyTorch.
-
-### Changed
-- **Parameter `alpha_ss`**: Removed. In 1.3.3, `alpha_ss=0.5` was used for soft-sign scaling $z / (1 + \alpha _ {\text{ss}} |z|)$ in the legacy single-buffer path. In 1.5.3, TD-CLM envelope normalization bounds updates directly to $(-1, 1)$, making soft-sign obsolete.
-- **Parameter `lam`**: Added with default `0.3`. Governs the defect penalty weight in TD-CLM.
-- **Parameter `centered_defect`**: Added with default `False`. Enables batch-mean centering of the innovation defect.
-- **Parameter `bound_type`**: Default changed from `"radial"` (1.3.3) to `"coordinate"` (1.5.3). Coordinate-wise hyperbolic bounding $(B_i \tanh(\Delta \theta_i / B_i))$ evaluates locally on each parameter without computing a tensor-wide norm reduction.
-- **Parameter `bound_ratio`**: Default changed from `0.03` to `0.05` to provide better update headroom on large model architectures.
-- **Parameter `ema_decay`**: Default changed from static `0.8` to `"auto"`. Resolves to `0.80` for Dual-Buffer mode (`buffer=2`) and `0.95` for Single-Buffer mode (`buffer=1`).
-- **Scratch Buffer**: Expanded from 5 floats `[mask_sum, u_sq_sum, p_sq_sum, cautious_scale, bound_scale]` to 6 floats, adding `diff_sum` at index 5 to accumulate batch-mean defect statistics.
-- **Cautious Scaling Floor**: Enforced `cautious_clamp_min=0.5` across all backends to cap maximum cautious energy scaling at $2.0\times$ ($2^1$).
-
-### Fixed
-- **Hardware Mantissa Divergence**: Fixed bitwise output differences across GPU families by replacing hardware SFU `tl.math.tanh` with Padé [13/6] rational approximation.
-- **FP16 Subnormal Bit Corruption**: Fixed exponent corruption in subnormal FP16 stochastic rounding by separating exponent extraction from significand shift logic.
-- **Single-Buffer Kernel Launch Latency**: Reduced single-buffer kernel dispatches from 3 passes to 1 pass, eliminating host-side synchronization bubbles and intermediate L2 atomic reduction overhead.
-- **Non-Zero Mean Gradient Drift**: Resolved directional accumulator drift on bias and normalization layers via `centered_defect`.
-
-### Parameter Comparison (1.3.3 vs 1.5.3)
-
-| Parameter | Version 1.3.3 | Version 1.5.3 | Technical Notes |
-| :--- | :--- | :--- | :--- |
-| `params` | `params` | `params` | Iterable of parameters or parameter groups. |
-| `lr` | `8e-4` | `8e-4` | Base learning rate. |
-| `betas` | `(0.9, 0.999)` | `(0.9, 0.999)` | First and second moment exponential decay rates. |
-| `eps` | `1e-8` | `1e-8` | Numerical stability floor. |
-| `weight_decay` | `8e-2` | `8e-2` | Decoupled weight decay rate. |
-| `tau` | `0.8` | `0.8` | Noise dispersion scale factor. |
-| `alpha_ss` | `0.5` | *(removed)* | Removed; superseded by TD-CLM envelope normalization. |
-| `lam` | *(none)* | `0.3` | Defect penalty weight in TD-CLM (calibrated to $\lambda^* \approx 0.3008$). |
-| `cautious` | `True` | `True` | Directional verification gate ($u_i g_i > 0$). |
-| `cautious_clamp_min` | `0.5` | `0.5` | Maximum cautious scale cap ($2.0\times$). |
-| `buffer` | `2` | `2` | `1` = Single-Buffer (TD-CLM), `2` = Dual-Buffer (CLM). |
-| `centered_defect` | *(none)* | `False` | Enables subtraction of batch-mean defect $\mu_d$. |
-| `stochastic_rounding` | `True` | `True` | Bitwise stochastic rounding on FP16/BF16 weights. |
-| `bound` | `True` | `True` | Enables update trust-region bounding. |
-| `bound_type` | `"radial"` | `"coordinate"` | Default changed to coordinate-wise bounding (zero reduction overhead). |
-| `bound_ratio` | `0.03` | `0.05` | Maximum step magnitude relative to parameter magnitude. |
-| `master_weights` | `False` | `"none"` | `"none"`, `"semi"`, or `"full"`. |
-| `execution` | `"auto"` | `"auto"` | Dynamic routing: SIMD, Vulkan, Triton, or Foreach. |
-| `fused_single_pass` | `False` | `True` | Enables single-pass streaming with EMA scaling. |
-| `ema_decay` | `0.8` | `"auto"` | Automatically sets `0.80` for dual-buffer, `0.95` for single-buffer. |
-| `fsdp_sync` | `False` | `False` | Cross-rank collective all-reduce synchronization. |
-| `seed` | `1337` | `1337` | PRNG seed for deterministic stochastic rounding. |
-
----
-
-## [1.3.3] - 2026-10-06
-
-### Fixed
-* **Triton AST Compiler Early-Return Inlining Error:**
-  * Resolved fatal MLIR/CFG code generation failure (`at 35:4: _store_state_buffer(...) ^`) triggered by mid-branch `return` statements in `@triton.jit` helper functions (`_store_param` and `_store_state_buffer`).
-  * Refactored helper control flow into structured, single-exit `if ... elif ... else` branches, ensuring compliant JIT function inlining across all kernel entry points.
-* **Bitcast Width Mismatch (Int64 Promotion in Stochastic Rounding):**
-  * Fixed implicit 64-bit integer promotion caused by Python integer literals (`0x9E3779B9`, `~0x1FFF`, `~0xFFFF`) inside Triton bitwise operations.
-  * Enforced strict 32-bit signed arithmetic via two's complement constants (`-1640531527`, `-2048173461`, `-1028477387`) with explicit `.to(tl.int32)` coercion, and standardized bitwise masks (`-65536` for BF16, `-8192` for FP16), ensuring type-safe 32-bit bitcasting (`tl.int32` to `tl.float32`).
-* **JIT Kernel Call Keyword Arguments:**
-  * Replaced internal keyword argument invocations (`use_sr=False`) with standard positional boolean parameters inside Triton JIT kernels to eliminate compiler argument binding warnings across Triton runtime versions.
-
-### Improved
-* **Throttled Fallback Telemetry & Console De-Noising:**
-  * Added `self._warned_triton_failure` state tracking to prevent multi-line compiler AST dumps from flooding the console across every parameter tensor in a parameter group.
-  * Formatted fallback notifications into a single concise one-line `WARNING` log on initial encounter, routing full tracebacks to `DEBUG` level.
-
-### Verified
-* **End-to-End JIT Execution on NVIDIA Architectures (Tesla T4):**
-  * Validated JIT execution across both Dual-Pass (`fused_single_pass=False`) and Fused Single-Pass (`fused_single_pass=True`) modes in native FP16.
-  * Confirmed non-zero weight update dynamics ($|\Delta W| > 0$) without gradient underflow or weight freezing in pure master-free training for Version 1.3.3.
-
----
-
-## [1.3.2] - 2026-10-06
-
-### Added
-- **Distributed FSDP Collective Synchronization (`fsdp_sync`)**:
-  - Introduced the `fsdp_sync: bool = False` configuration flag on optimizer initialization, backed by automated runtime guarding via `torch.distributed.is_initialized()`.
-  - Integrated cross-GPU collective communications (`torch.distributed.all_reduce(..., op=torch.distributed.ReduceOp.SUM)`) for global reduction metrics (`mask_sum`, `u_sq_sum`, and `p_sq_sum`) across Triton kernels, single-step execution, and vectorized foreach loops.
-  - Maintains bit-exact scalar consistency for `m_bar`, `cautious_scale`, and `bound_scale` across distributed parameter shards during large-scale pretraining under PyTorch FSDP (Fully Sharded Data Parallel) and DeepSpeed ZeRO-3.
-- **Dedicated Low-Precision Stochastic Quantization Routine (`_sr_quantize`)**:
-  - Factored out bitwise stochastic rounding into a dedicated, modular helper function: `_sr_quantize(target, val_f32, use_sr, generator)`.
-  - Enabled native stochastic rounding on half-precision momentum states (`exp_avg` in BF16/FP16) across all PyTorch CPU/CUDA fallback paths (`_single_step_v2`, `_foreach_step_v2`, `_single_step_v1`, and `_foreach_step_v1`), systematically eliminating gradient stagnation induced by fractional bit truncation (LSB underflow).
-- **Persistent Contiguous Buffer Caching**:
-  - Introduced persistent state cache tensors within `self.state[p]` (`p_contig`, `exp_avg_contig`, `exp_avg_sq_contig`, and `mp_contig`) to stage strided or sliced parameters before kernel launch.
-  - Eliminates dynamic memory allocations (`.contiguous()`) during iterative training loops, preserving PyTorch VRAM caching allocator stability and preventing CUDA Graph capture invalidation.
-- **Zero-Copy Direct Tensor Aliasing for FP32 Master Weights**:
-  - When parameter tensors are natively `torch.float32`, `state["master_param"]` now directly aliases the parameter tensor pointer (`state["master_param"] = p`), avoiding additional master weight VRAM overhead for FP32 layers (0 bytes auxiliary memory footprint).
-- **Checkpoint Sanitizer via State Dict Filter (`state_dict`)**:
-  - Overrode `state_dict()` to automatically purge temporary contiguous cache buffers (`p_contig`, `exp_avg_contig`, `exp_avg_sq_contig`, `mp_contig`), ensuring on-disk checkpoint payloads remain clean, compact, and fully backward-compatible.
-- **Defensive Device Pipeline Flush on Kernel Fault (`torch.cuda.synchronize`)**:
-  - Embedded explicit hardware device synchronization (`torch.cuda.synchronize(p.device)`) within Triton exception recovery handlers prior to PyTorch fallback execution, preventing asynchronous CUDA error cascades into subsequent tensor operations.
-
-### Changed
-- **High-Entropy Bit-Avalanche PRNG Hash Function (`_prng`)**:
-  - Replaced the standard Linear Congruential Generator (LCG) with a MurmurHash3 / SplitMix32 integer bit-avalanche hashing routine:
-    ```python
-    z = idx + seed + 0x9E3779B9
-    z = (z ^ (z >> 16)) * 0x85EBCA6B
-    z = (z ^ (z >> 13)) * 0xC2B2AE35
-    z = z ^ (z >> 16)
-    ```
-  - Delivers uniform pseudo-random noise distributions with zero low-order bit periodicity, significantly refining stochastic rounding fidelity on BF16/FP16 parameters.
-- **Memory-Contiguous Scratch Buffer Realignment for v1 Triton Kernels**:
-  - Re-indexed reduction buffer offsets in Triton v1 kernels:
-    - Pass 1 metrics (`[rms_sum, p_sq_sum]`) mapped to contiguous indices `[0, 1]`.
-    - Pass 2 metrics (`[mask_sum, u_sq_sum]`) mapped to contiguous indices `[2, 3]`.
-  - Enables vectorized contiguous slice all-reduces (`scratch_param[:2]` and `scratch_param[2:4]`) under FSDP without requiring non-contiguous memory gather/scatter overhead.
-- **Multi-GPU Device Index Resolution in RNG Dispatcher (`_get_generator`)**:
-  - Fixed device index resolution when `device` is passed as a generic `torch.device("cuda")` without an explicit index by defaulting to `torch.cuda.current_device()`, avoiding cross-GPU generator key collisions during multi-GPU distributed runs.
-- **Conditional Atomic Reduction Pruning in v2 Pass 1 Kernel**:
-  - Guarded `mask_sum` atomic accumulation in `_lumina_v2_pass1_kernel` with `if cautious:`, bypassing CTA reductions and L2 atomic additions when `cautious=False`.
-- **Expanded Master Weight Precision Configuration Aliases**:
-  - Added the `"half"` alias for `"semi"` mode alongside `"fp32"` and `"masterweight"` aliases for `"full"` mode.
-- **Metadata & Documentation Header Update**:
-  - Bumped specification version to `1.3.2`.
-
-### Fixed
-- **Singular Zero-Division & Step-Zero Bias Correction (`safe_base_step`)**:
-  - Introduced `safe_base_step = max(base_step, 1)` and `bc2_base = max(1.0 - (beta2 ** safe_base_step), 1e-15)` across Triton, single-step, and foreach loops.
-  - Fully resolves potential `ZeroDivisionError` and NaN blowups in `c1` / `c2` computations when steps evaluate at `step == 0`.
-- **Numerical Overflow Clamp in Sigmoid-Based Tanh Fallback**:
-  - Applied register clamp bounds `x_clamped = tl.maximum(tl.minimum(x, 20.0), -20.0)` in analytical fallback `_triton_tanh_fast`, preventing `tl.exp(-2.0 * x)` from overflowing to `inf` or `nan` on devices lacking hardware SFU tanh instructions.
-- **Missing Argument Forwarding in PyTorch Single-Step v1**:
-  - Corrected `_single_step_v1` method signature and call-sites to accept and propagate `scratches`, eliminating runtime `TypeError` during fallback invocation.
-- **Multi-Rank Synchronization Skew in Sharded Parameter Bounding**:
-  - Aligned tensor norms and radial bounding factors across sharded ranks via synchronized global scalar reductions, preventing parameter update drift across distributed ranks.
-
-### Performance & Engine Refactoring
-- **Zero Dynamic VRAM Allocation on Strided / Sliced Tensors**:
-  - Persistent pre-allocated contiguous staging buffers avoid PyTorch caching allocator churn and eliminate CUDA Graph memory address invalidations.
-  - Preserves deterministic memory footprints across long pre-training runs.
-- **Optimized Distributed Collective Bandwidth**:
-  - Grouping scalar reductions into contiguous vector slices compresses distributed synchronization overhead into a single collective kernel launch per pass.
-
----
-
-## [1.3.0] - 2026-09-23
-
-### Added
-- **Fused Single-Pass Kernel with Post-Step EMA (`fused_single_pass`, `ema_decay`)**:
-  - Introduced `_lumina_v2_fused_single_pass_kernel` coupled with an asynchronous CTA micro-kernel `_lumina_v2_post_step_ema_kernel[(1,)]`.
-  - When `fused_single_pass=True`, avoids two-pass global reductions by leveraging running Exponential Moving Average (EMA) estimates of `cautious_scale` ($1 / \bar{M}$) and `bound_scale` ($\tanh(r) / r$).
-  - Unifies gradient filtering, momentum accumulation, variance dispersion, cautious masking, radial bounding, decoupled weight decay, stochastic rounding, and block partial reductions into a **single unified VRAM-resident pass**, reducing global GPU memory bandwidth traffic per optimizer step.
-- **Hybrid Master Weight Precision Architecture (`master_weights`)**:
-  - Added multi-tier master weight flexibility via `master_weights: Union[bool, str] = False`:
-    - `"none"` / `False` *(Default)*: Pure master-free low-precision training leveraging on-chip bitwise stochastic rounding directly on native half-precision weights.
-    - `"semi"` / `"hybrid"` / `"half"`: 32-bit FP32 master parameters (`state["master_param"]`) coupled with 16-bit low-memory optimizer states (`exp_avg`, `exp_avg_sq` in BF16/FP16), saving ~50% optimizer state VRAM compared to traditional FP32 setups while maintaining full FP32 weight accumulation accuracy.
-    - `"full"` / `"fp32"` / `"masterweight"`: Traditional full FP32 master parameters with full FP32 optimizer states for standard baseline comparisons.
-- **In-Kernel In-Flight NaN / Inf Gradient Sanitization**:
-  - Embedded register-level defensive sanitization across all Triton GPU kernels: `is_finite_g = (g == g) & (g < 1e38) & (g > -1e38); g = tl.where(is_finite_g, g, 0.0)`.
-  - Replaces non-finite gradients and subnormal corruptions with zero in registers before they can contaminate momentum (`exp_avg`) or variance (`exp_avg_sq`) state buffers.
-- **Low-Precision State Buffer Serialization (`_store_state_buffer`)**:
-  - Added a dedicated Triton state serialization routine supporting FP32, BF16, and FP16 state representations with optional bitwise stochastic rounding applied directly to momentum buffers.
-- **Multi-Device & Cross-Accelerator Hardware Context Isolation (`_get_device_context`)**:
-  - Unified device stream scoping supporting NVIDIA CUDA (`torch.cuda`), Intel XPU (`torch.xpu`), and modern PyTorch 2.x unified accelerator interfaces (`torch.accelerator.device`), eliminating multi-GPU stream hijacking and cross-device race conditions.
-- **Strict Determinism Auto-Routing**:
-  - Automatically queries `torch.are_deterministic_algorithms_enabled()`. When enabled under `execution="auto"`, dynamically routes execution to C++ vectorized `foreach` to preserve bitwise cross-run reproducibility, circumventing non-deterministic floating-point hardware atomic reordering.
-- **Granular Fault Recovery with State Guarding (`completed_indices`)**:
-  - Implemented per-parameter completion tracking in `_dispatch_triton`. If a kernel encounters an unexpected runtime fault, completed parameters are preserved while uncompleted ones fall back cleanly to `foreach`.
-  - Added `_single_step_v2_apply_param` and `_single_step_v1_apply_param` to safely finalize parameter updates if Pass 1 had already committed state transitions, strictly preventing double-update state corruption.
-
-### Changed
-- **Per-Parameter Persistent Scratch Tensors (Elimination of Global Slicing)**:
-  - Replaced the global dynamic slice allocator (`self._scratch_tensors[dev][:count]`) with isolated per-parameter persistent state allocations (`state["scratch"] = torch.zeros((5,), device=p.device, dtype=torch.float32)`).
-  - Memory pointers for `[mask_sum, u_sq_sum, p_sq_sum, cautious_scale, bound_scale]` remain completely invariant throughout the entire training lifecycle, preserving pointer stability for **CUDA Graph Capture** (`torch.accelerator.Graph`) and TorchInductor AOT compilation without memory aliasing.
-- **Analytically Exact Nesterov Bias Correction ($\beta_1^{\text{step} + 1}$)**:
-  - Corrected first-moment bias correction from the classical Adam formulation ($1.0 - \beta_1^{\text{step}}$) to the exact analytical expectation of the Nesterov blended momentum: `bc1_nes = 1.0 - beta1^(step + 1)`. Removes residual momentum bias during early warmup iterations.
-- **Adaptive CTA Tile Sizing and Dynamic Thread Warp Scaling**:
-  - Introduced hardware-aware CTA tile scaling based on parameter volume: parameters with $\ge 2,097,152$ elements (2M+) dynamically configure `BLOCK_SIZE = 2048` and `num_warps = 8` (versus `BLOCK_SIZE = 1024`, `num_warps = 4` for smaller tensors).
-  - Cuts total thread-block grid counts and L2 cache atomic addition pressure by 50% on large MLP, Attention Projection, and Vocabulary layers.
-- **Hardware-Native Tanh Instruction Dispatch (`tl.math.tanh` / `tl.tanh`)**:
-  - Upgraded `_triton_tanh_fast` to dynamically inspect compiler capabilities, binding directly to native PTX Special Function Unit (SFU) instructions (`tanh.approx.f32`) on modern Triton compilers while retaining analytical sigmoid fallbacks for legacy environments.
-- **PRNG Decorrelation between Momentum and Parameter Updates**:
-  - Separated RNG seed streams using bitwise alternating mask shifts: `seed_p = (seed + 0x55555555) & 0x7FFFFFFF`, mathematically eliminating cross-coupling noise correlation between state buffer rounding and parameter update rounding.
-- **Bumped specification version metadata to 1.3.0** across `luminav.py`.
-
-### Fixed
-- **Floating-Point Cancellation in Radial Bounding ($r \to 0$)**:
-  - Fixed precision jitter and roundoff degradation near zero by adding an asymptotic Taylor-series guard across Triton and PyTorch bounding engines: `torch.where(r < 1e-4, 1.0, tanh(r) / r)`.
-- **Artificial Denominator Dilation in Dispersion Scale ($\sigma$)**:
-  - Replaced additive epsilon inflation (`sigma + 1e-6`) with a non-distorting floor clamp (`tl.maximum(sigma, 1e-12)`), preventing artificial dampening of subtle gradient signals.
-- **Singular Zero-Division in Parameter Norm Clamping**:
-  - Replaced `norm_p + 1e-6` with `torch.clamp_min(norm_p, 1.0)` / `tl.maximum(norm_p, 1.0)`, maintaining numerical stability even when parameters are initialized near zero.
-- **Transient Memory Pool Spikes during PyTorch Fallback Stochastic Rounding**:
-  - Refactored `_sr_update` to stream stochastic noise generation in bounded chunks of 1,048,576 elements (~4 MB chunk size), eliminating multi-hundred megabyte temporary int32 tensor spikes during fallback execution on large parameter matrices.
-- **Non-Contiguous Buffer Safety & Strided Memory Copy**:
-  - Added strict contiguous layout verification for parameters, states, and master weights with in-place synchronization (`p.copy_(p_contig)`), guarded by a one-time developer warning.
-
-### Performance & Engine Refactoring
-- **~50% Global Memory Bandwidth Reduction via Fused Single-Pass Mode**:
-  - By unifying parameter loading, gradient evaluation, state transformation, and bounding into one GPU pass under `fused_single_pass=True`, VRAM read/write cycles are halved for cautious and radially bounded steps.
-- **Sub-Microsecond L2 Hardware Atomic Reduction Architecture**:
-  - Formally documented the hybrid Intra-Block Tree Reduction + Inter-Block Hardware Atomic pattern. With `BLOCK_SIZE=2048`, a 10M-parameter tensor issues only 4,883 atomic additions directly into GPU L2 cache crossbars, executing in < 0.5 µs and strictly outperforming multi-tier cascaded reduction micro-kernels by eliminating 3–5 µs driver dispatch bubbles.
-- **Static Buffer Address Invariance for CUDA Graphs & TorchInductor AOT**:
-  - Persistent 5-slot state scratch buffers eliminate all dynamic slicing and memory reallocations during optimization loops, preserving pointer address invariance for CUDA Graph replay.
-
----
-
-## [1.2.1] - 2026-09-17
-
-### Added
-- **In-Kernel GPU Parameter Norm Reduction (`p_sq_sum_ptr`)**: Fused parameter Euclidean norm squared $\sum(p_i^2)$ calculation directly into Pass 1 Triton reduction kernels (`_lumina_v2_pass1_kernel` and `_lumina_v1_pass1_kernel`). Pass 2 and Pass 3 now load and resolve `norm_p` (`tl.sqrt(tl.maximum(p_sq_sum, 0.0))`) entirely inside GPU SRAM/registers without host intervention.
-- **Stateless CPU Integer Multiplicative PRNG Hashing**: Replaced dynamic GPU RNG tensor allocation with a pure CPU integer hash using Knuth's 32-bit golden ratio constant: `(step * 0x9E3779B9 + i * 10007) & 0x7FFFFFFF`. Generates uncorrelated 31-bit integer seeds per layer with zero GPU kernel launch overhead and zero PCIe host-device latency.
-- **Isolated Generator Cache for Fallback Engine (`self._generators`)**: Added an internal per-device `torch.Generator` cache, ensuring stochastic rounding in `_sr_update` operates in complete isolation from PyTorch's default CUDA generator state.
-
-### Changed
-- **Consolidated Scratch Buffer Zeroing**: Removed redundant per-parameter slice zeroing inside parameter loops. Buffer zeroing is now batched into a single consolidated `scratch.zero_()` call per step inside `_get_scratch_buffer`.
-- **Scratch Buffer Stride Expansion**:
-  - `buffer=2`: Expanded stride from 2×N to 3×N elements per parameter to host `[mask_sum, u_sq_sum, p_sq_sum]`.
-  - `buffer=1`: Expanded stride from 3×N to 4×N elements per parameter to host `[rms_sum, mask_sum, u_sq_sum, p_sq_sum]`.
-- **Bumped specification version metadata to 1.2.1**.
-
-### Fixed
-- **Host-Device Synchronization & Pipeline Stalls (`.item()` Bottleneck)**: Removed `norm_p = float(p_contig.float().norm().item())` and `seed = torch.randint(...).item()` from Triton step pipelines. Eliminated 200–400 synchronous PCIe roundtrips and GPU pipeline stalls per step on deep architectures (e.g., 28-layer Transformers), restoring full asynchronous GPU compute utilization.
-- **CUDA Graphs & `torch.compile` Graph Breaks**: Fixed fatal graph capture crashes and TorchDynamo graph breaks caused by blocking host-device synchronization during radial bounding and seed generation. The optimizer step is now compatible with CUDA Graphs (`torch.accelerator.Graph`) and TorchInductor compilation.
-- **Transient VRAM Allocator Thrashing**: Eliminated repeated temporary FP32 parameter cloning (`p_contig.float()`) during norm evaluation, preventing memory pool fragmentation in tight VRAM regimes.
-- **Global RNG Pollution & Determinism Compliance**: Fixed global RNG state mutation in fallback stochastic rounding. Solved reproducibility drift in upstream Dropout, DataLoader shuffling, and RLHF reward modeling, ensuring strict adherence to PyTorch 2.10+ `torch.use_deterministic_algorithms(True)`.
-
-### Performance & Engine Refactoring
-- **Batch Kernel Launch Overhead Elimination**: Removed up to 400–800 individual `cudaMemsetAsync` micro-calls per step previously triggered by per-slice `.zero_()` invocations on 1-element scratch tensors.
-- **Single-Pass Bypass for Coordinate Bounding**: Streamlined `_triton_step_v2` to immediately bypass two-pass reductions and invoke `_lumina_v2_single_pass_kernel` when `cautious=False` and coordinate bounding (`bound_mode == 2`) is selected.
-
----
-
-## [1.2.0] - 2026-09-17
-
-### Added
-- **Automatic Dynamic FP16 Cliff Governor:** Built-in asymptotic boundary governor (`_triton_tanh_fast` in GPU register and PyTorch fallback) that automatically activates when `p.dtype == torch.float16`. Prevents unclipped steps from pushing parameters, activations, and residual streams into the IEEE-754 FP16 overflow cliff (> 65,504). Remains completely dormant (zero overhead) on `bfloat16` (TPU / Ampere+) and `float32`.
-- **Smooth Asymptotic Parameter Bounding (`bound`, `bound_type`, `bound_ratio`):** Added non-intrusive step bounding with two mathematically continuous formulations:
-  - `"radial"`: Direction-Preserving Radial Squashing (`update * tanh(r) / r`) preserving gradient direction (Cosine Similarity = 1.000).
-  - `"coordinate"`: Dual-Hyperbolic Coordinate Squashing for fine-grained per-element damping.
-  - Configurable via constructor: `bound: bool = True`, `bound_type: str = "radial"`, `bound_ratio: float = 0.03`.
-
-### Changed
-- **Increased Default `cautious_clamp_min` from `0.2` to `0.5`:** Enforces an exact power-of-two maximum energy scaling factor (2.00x, $2^1$). Eliminates floating-point mantissa roundoff distortion in IEEE-754 hardware.
-- Bumped specification version metadata to `1.2.0`.
-
-### Fixed
-* **FP16 Step Divergence & NaN Overflow at High Learning Rates:** Solved step divergence and subsequent NaN crashes when training deep 28-layer Transformer architectures (e.g., Qwen3-0.6B) in pure FP16 at nominal learning rates (`lr = 8e-4`).
-* **Sparse Embedding & LM-Head Amplification:** Mitigated 5x over-amplification on sparse vocabulary gradients where active token counts are much smaller than total tensor parameters ($N \gg N_{\text{active}}$).
-* **Verified stable 1,000-step pretraining from scratch on Qwen3-0.6B** (seq length 512, pure FP16, `lr = 8e-4`) on Nvidia Tesla T4 with zero gradient clipping and zero LR scheduler (loss dropped from 12.09 down to ~5.7 with zero NaNs).
-* **Weight Decay Numerical Parity on Fallback Engine:** Resolved a silent precision-stagnation issue in `_single_step` and `_foreach_step`. Previously, decoupled weight decay (`p.mul_(1.0 - lr * weight_decay)`) was executed directly on native FP16/BF16 tensors prior to `_sr_update`. Due to IEEE-754 Round-to-Nearest (RTN) limits, `(1 - lr * weight_decay) ≈ 0.999936` rounded back to `1.0`, causing weight decay to freeze on half-precision parameters. Weight decay is now unified into FP32 registers inside `_sr_update` alongside stochastic rounding, matching the numerical behavior with the Triton kernel.
-* **Fallback Foreach Indentation & Scope Alignment:** Corrected tensor iteration scope and indentation for `torch._foreach_add_` in non-SR fallback branches.
-* **In-Place GPU Clamping in `_apply_bound`:** Replaced dynamic GPU scalar allocations (`torch.tensor(1.5e-4, device=p.device)`) with native `torch.clamp_min`, eliminating transient CUDA malloc spikes per update step.
-
-### Performance & Engine Refactoring
-* **Unified Bound Execution Pipeline:** Added an immediate short-circuit guard (`if lr == 0.0: return update`) and streamlined the execution flow: intermediate updates are scaled in FP32 (`step_val = lr * u_float`), passed through radial/coordinate bounding, chained cleanly into the FP16 Cliff Governor, and returned in a single normalized step.
-
----
-
-## [1.1.5] - 2026-09-15
-
-### Added
-- **Native Hardware Capability Detection (`_supports_native_bf16`):** Added automatic device capability detection via `torch.cuda.is_bf16_supported()` to identify Ampere SM80+ architectures and streamline native execution paths.
-- **Hugging Face Repository Reference:** Added official Hugging Face repository link in module docstrings.
-
-### Changed
-- **Dynamic Epsilon Clamping for FP16:** Dynamically floors effective epsilon to `max(eps, 1e-4)` when parameters or states are in `torch.float16`. Prevents bias-corrected scale factor `c2` from underflowing into IEEE-754 subnormal/zero limits on step 1.
-- Bumped specification version to `1.1.5`.
-
-### Fixed
-- **FP16 Step-1 `NaN` from Epsilon Underflow:** Fixed `0.0 / 0.0 = NaN` division occurring on initial training steps when parameters receive zero gradients (e.g., unselected vocabulary embeddings, padding tokens, or dropout paths).
-- **Second-Moment Variance Saturation (`+inf`):** Added safe upper-bound register clamping (`tl.minimum(v_new, 65000.0)` in Triton kernels and `.clamp_(0.0, 65000.0)` in PyTorch C++ foreach/single loops). Prevents unclipped initial pretraining gradient surges ($|g| > 256$) from overflowing the FP16 maximum dynamic range (65,504) into irreversible `+inf`.
-- **Zero-Sigma Division Guard:** Injected explicit `sigma + 1e-6` denominator guards across all Triton kernel passes and PyTorch update steps to ensure numerical safety during cold-start iterations.
-- Verified stable 100-step pretraining from scratch on `SmolLM-135M` using Wikimedia Wikipedia in pure FP16 on Nvidia Tesla T4 GPU with zero gradient clipping.
-
----
-
-## [1.1.0] - 2026-09-11
-
-### Added
-- **Bitwise Stochastic Rounding (SR):** Integrated on-chip PRNG integer bit-manipulation hash in Triton kernels and vectorized PyTorch fallback for native FP16/BF16 training without master weights.
-- Unified Triton store routine introducing compile-time branching (`dtype_mode` / `use_sr: tl.constexpr`) with explicit safe casting for FP16 and BF16 pointers.
-- Vectorized fallback improvements utilizing C++ multi-tensor operations when stochastic rounding is disabled or tensors are FP32.
-- State buffer contiguity validation (`exp_avg_contig`, `exp_avg_sq_contig`) ensuring safe linear memory addressing before kernel dispatch.
-- Declarative configuration file (`config.json`) defining hyperparameters and optimizer presets.
-
-### Changed
-- Refactored PRNG integer arithmetic to strictly stay within signed 32-bit integer limits ($< 2^{31} - 1$), eliminating JIT compilation failures on newer Triton releases.
-- State buffers (`exp_avg`, `exp_avg_sq`) now initialize with strict contiguous memory format.
-
-### Fixed
-- Fixed `ValueError: Scalar out of range for type int32` and Triton LLVM compile errors during unsigned 32-bit bitmask operations.
-- Resolved kernel fallback hangs and compilation overhead on CUDA devices.
-
----
-
-## [1.0.2] - 2026-09-08
-
-### Added
-- Official Apache-2.0 license notice and copyright headers.
-- Standardized package exports (`__all__`, `__version__`, `__author__`, `__license__`).
-
----
-
-## [1.0.1] - 2026-08-31
-
-### Fixed
-- **Memory Synchronization Fix (`423d171`):** Fixed a critical bug where non-contiguous parameter tensors (e.g., transposed linear layers or sliced weights) were updated on temporary contiguous copies without reflecting changes back to model weights. Added `p.copy_(p_contig)` after Triton execution.
-
----
-
-## [1.0.0] - 2026-08-30
-
-### Added
-- Initial release of the LuminaV optimizer.
-- **Zero Master-Weight Copies:** Direct parameter updates in native half-precision without allocating 4-byte FP32 master weights.
-- **Hyperbolic Tangent (`tanh`) Bounding Envelope:** Projects normalized momentum ratio into `(-1.0, 1.0)` to eliminate exploding gradient shocks.
-- **Directional Alignment Gate:** Cautious update masking that zeroes out momentum steps opposing mini-batch gradient directions.
-- **Central Innovation Variance:** Noise dispersion tracking $(g_t - m_t)^2$ decoupling mini-batch turbulence from directional descent.
-- Dual-buffer mode (`buffer=2`, LuminaV-2B) and single-buffer mode (`buffer=1`, LuminaV-1).
-- Custom fused OpenAI Triton kernels with automated fallback execution.
+# LuminaV Optimizer Algorithm Changelog
+
+All notable changes to the LuminaV Optimizer algorithm are documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+* [Software Package Changelog (`luminamoon`)](SOFTWARE_CHANGELOG.md)
+
+---
+
+## [1.6.0] - 2026-11-03
+
+### Added
+- **T-Family (Triangle Family) tanh-free Modes (`buffer="T1"` & `buffer="T2"`)**:
+  * **`buffer="T2"` (2-Buffer Triangle Family Mode)**: Keeps the 2-buffer state (`exp_avg` and `exp_avg_sq`) without using $\tanh$, computing the gradient-momentum residual $d_t = g_t - m _ {t-1}$ and bounding the lookahead update $\tilde{m}_t$ via the **Triangle Inequality Envelope** $\mathcal{E}_t^{(\text{T2})}$:
+
+    $\displaystyle d_t = g_t - m _ {t-1}, \qquad v_t = \beta_2 v _ {t-1} + (1 - \beta_2) d_t^2$
+
+    $\displaystyle \tilde{m} _ t = w_1 m _ {t-1} + w_2 g_t, \qquad w_1 = \beta_1^2, \qquad w_2 = 1 - \beta_1^2$
+
+    $\displaystyle \mathcal{E} _ t^{(\text{T2})} = w_1 |m _ {t-1}| + w_2 |g_t| + 0.5\lambda |d_t| + 0.5\lambda \sigma_t + \epsilon _ {\text{eff}}, \qquad u_t^{(\text{T2})} = \frac{\tilde{m} _ t}{\mathcal{E} _ t^{(\text{T2})}} \in (-1, 1)$
+
+    By the Triangle Inequality ($|w_1 m _ {t-1} + w_2 g_t| \le w_1 |m _ {t-1}| + w_2 |g_t| \le \mathcal{E} _ t^{(\text{T2})}$), $|u_t^{(\text{T2})}| < 1$ is guaranteed without evaluating $\tanh$. The term $0.5\lambda |d_t|$ acts as a **triangle-defect penalty**, and $0.5\lambda \sigma_t$ provides belief-variance damping.
+  * **`buffer="T1"` (1-Buffer Triangle Family Mode)**: Combines the single-buffer **Triangle-Defect** denominator $\mathcal{E} _ t^{(\text{T1})} = w_1 |m _ {t-1}| + w_2 |g_t| + \lambda |d_t| + \epsilon _ {\text{eff}}$ (using residual $d_t = g_t - m _ {t-1}$) with the rational update bounds below, reducing optimizer state memory by 50% via the Triangle Inequality.
+  * **Rational Update Bounds (for `T1` & `T2`)**: Replaces $\tanh$ and $\sqrt{\cdot}$ in the coordinate, radial, and FP16 update bounds with simple rational formulas:
+
+    $\displaystyle b_i = \mathrm{bound} _ {\text{ratio}} \cdot (|w_i| + 10^{-2}), \qquad \Delta w_i = \frac{b_i (b_i + |s_i|) s_i}{b_i (b_i + |s_i|) + s_i^2}, \qquad \mathrm{scale} _ {\text{radial}} = \frac{1 + r}{1 + r + r^2}$
+  * Keeps 100% backward compatibility and identical outputs for existing modes (`buffer=1`, `buffer=2`, and `version="1.5.5"`).
+
+---
+
+## [1.5.5] - 2026-10-29
+
+### Added
+- **Contiguous Bucket Fusion (`fuse_tensors: bool = False`)**:
+  * Added opt-in contiguous parameter and state coalescing (`_build_contiguous_buckets`) that groups parameters within each `(param_group, device, dtype)` domain $\mathcal{B}_{g,d,\tau}$ into contiguous 1D storage (`flat_p`, `flat_g`, `flat_m`, `flat_v`, `flat_mp`) and rebinds `p.data`, `p.grad`, and `self.state[p]` entries as zero-copy views (`flat_*[offset : offset + n].view(p.shape)`).
+  * Eliminates per-tensor CPU-to-GPU kernel launch latency on fragmented vision and edge architectures by executing an entire bucket in a single fused kernel dispatch while computing `cautious_scale` and `radial` trust bounds across the bucket domain $\mathcal{B}_{g,d,\tau}$.
+  * Added in-place `zero_grad()` view preservation (`flat_g.zero_()`) and full `state_dict()` / `load_state_dict()` checkpoint compatibility without leaking temporary buffer keys (LSO-Eval Stage II compliant).
+- **Triton JIT Pre-Warmup (`prewarm_jit: bool = True`)**:
+  * Added automatic initialization-time Triton JIT compilation (`_prewarm_all_configs`) using 64-element dummy buffers (`128–256 bytes` VRAM) with `lr=0.0` and `weight_decay=0.0` across both Step 1 (`pass1` + `pass2`) and Step 2+ (`fused_single_pass` + `post_step_ema`) paths with target `BLOCK_SIZE` specialization (`1024` or `2048`), eliminating first-step compilation stalls.
+- **Bounded Bucket Chunking & Sequential In-Place Migration (`bucket_cap_mb: float = 500.0`)**:
+  * Bound contiguous bucket coalescing by an upper memory capacity ceiling $C_{\text{cap}} = 500\text{ MB}$, partitioning model parameters into ordered contiguous chunks $\mathcal{B}^{(1)}, \mathcal{B}^{(2)}, \dots, \mathcal{B}^{(K)}$ such that $\sum_{p \in \mathcal{B}^{(k)}} \text{bytes}(p) \le C_{\text{cap}}$ (with giant single layers $> C_{\text{cap}}$ isolated into individual buckets).
+  * Enforced sequential in-place migration with immediate parameter storage recycling (`del old_data`, `del old_grad`) and garbage collection, constraining peak transient memory allocation spikes during optimizer initialization to at most $\min(\text{model bytes}, C _ {\text{cap}})$.
+
+---
+
+## [1.5.4] - 2026-10-29
+
+### Added
+- **In-Container Native Bitwise-BF16 State Promotion (`dtype_mode == 2`, `master_weights="none"`)**:
+  * Promoted 16-bit optimizer state buffers ($m_t$ via stochastic rounding `f32_to_bf16_bits_sr` and $v_t$ via round-to-nearest-even `f32_to_bf16_bits_rne`) to native Bitwise-BF16 stored inside the existing 16-bit container during pure `FP16` execution, expanding state exponent dynamic range to 8 bits ($1.18 \times 10^{-38}$) with zero additional VRAM overhead.
+
+### Changed
+- **Unified Denominator Stability Constant ($\epsilon_{\text{eff}} = \epsilon = 10^{-8}$)**:
+  * Unified the effective denominator constant $\epsilon_{\text{eff}} = \epsilon = 10^{-8}$ across all precision modes and hardware backends (replacing the legacy `FP16` floor clamp $\max(\epsilon, 10^{-4})$ introduced in `1.1.5`), preventing update velocity attenuation and late-stage convergence loss resets when $\sqrt{v_t} \ll 10^{-4}$.
+
+### Removed
+- **Centered Defect Compensation (`centered_defect`, $\mu_d$)**:
+  * Removed the `centered_defect` hyperparameter and the global defect mean reduction $\mu_d = \frac{1}{N}\sum_{i=1}^N (g_{t,i} - m_{t-1,i})$ across both Dual-Buffer Convex Lookahead Momentum (`buffer=2` / CLM) and Single-Buffer Triangle-Defect Convex Lookahead Momentum (`buffer=1` / TD-CLM).
+  * Simplified the instantaneous gradient-momentum residual to purely local coordinate-wise differences:
+
+    $\displaystyle d_t^{(\text{B1})} = g_t - m_{t-1}, \qquad d_t^{(\text{B2})} = g_t - m_t$
+
+  * Eliminated the pre-step global reduction pass (`_lumina_diff_sum_kernel` in Triton JIT/AOT, `diff_sum.comp` in Vulkan SPIR-V, `reduce_diff_sum_simd` in x86\_64 AVX-512/AVX2, and slot 5 of the 6-element `ScratchBuffer`), removing cross-lane/cross-rank reduction latency while retaining the 6-element physical `ScratchBuffer` layout (`[mask_sum, u_sq_sum, p_sq_sum, cautious_scale, bound_scale, _reserved]`, 5 active slots + 1 reserved slot for ABI alignment).
+
+---
+
+## [1.5.3] - 2026-10-20
+
+No versions were published between 1.3.3 and 1.5.3. This release notes all changes made since version 1.3.3.
+
+### Added
+- **Single-Buffer Triangle-Defect Convex Lookahead Momentum (TD-CLM, `buffer=1`)**:
+  * Replaced the legacy 3-pass RMS single-buffer update with a single-pass formulation:
+
+    $\displaystyle \tilde{m} _ t = w_1 m _ {t-1} + w_2 g_t, \quad w_1 = \beta_1^2, \quad w_2 = 1 - \beta_1^2$
+
+    $\displaystyle \mathcal{E} _ t = w_1 |m _ {t-1}| + w_2 |g_t| + \lambda |d_t^{(\text{B1})}| + \epsilon _ {\text{eff}}$
+
+    $\displaystyle u_t^{(\text{B1})} = \frac{\tilde{m} _ t}{\mathcal{E} _ t} \in (-1, 1)$
+
+  * In 1.3.3, single-buffer mode required three sequential GPU kernel launches (`_lumina_v1_pass1_kernel`, `_lumina_v1_pass2_kernel`, `_lumina_v1_pass3_update_kernel`) and global atomic reductions across thread blocks to compute tensor-wide root-mean-square ($\text{rms} = \sqrt{\frac{1}{N}\sum \tilde{m} _ i^2 + \epsilon}$).
+  * TD-CLM replaces the tensor-wide reduction with coordinate-wise envelope bounding via the Triangle Inequality Envelope $\mathcal{E} _ t$. This reduces single-buffer execution to a single GPU pass and eliminates all $\tanh$ evaluations in the momentum path.
+  * Calibrated the defect penalty weight $\lambda$ to the symmetric spike-reversal equilibrium:
+
+    $\displaystyle \lambda^* = \frac{2(1 - \beta_1^2)^2}{4\beta_1^2 - 3}$
+
+    For the default $\beta_1 = 0.9$, $\lambda^* \approx 0.3008$, setting the default parameter to `lam=0.3`.
+
+- **Padé [13/6] Rational Approximation (`tanh_pade_f32`)**:
+  * Implemented a rational Padé [13/6] polynomial for hyperbolic tangent calculations:
+
+    $\displaystyle \tanh _ {\text{Pade}}(x) = \frac{x(1 + c_1 x^2 + c_2 x^4 + c_3 x^6 + c_4 x^8 + c_5 x^{10} + c_6 x^{12})}{1 + d_1 x^2 + d_2 x^4 + d_3 x^6}$
+
+  * Evaluated over the clamped domain $x \in [-7.9988117, 7.9988117]$ with asymptotic saturation to $\pm 1.0$.
+  * In 1.3.3, `_triton_tanh_fast` used hardware-dependent SFU instructions (`tl.math.tanh`) or analytical sigmoid fallbacks (`2 / (1 + exp(-2x)) - 1`). Different GPU architectures (e.g. Turing, Ampere, Hopper) evaluate SFU transcendental functions with slightly different mantissa rounding, leading to bitwise divergence during distributed training.
+  * Padé [13/6] provides uniform precision with $\lVert \tanh _ {\text{Pade}}(x) - \tanh(x) \rVert _ \infty < 1.5 \times 10^{-6}$ across CPU Scalar, AVX2, AVX-512, Vulkan SPIR-V, and Triton GPU kernels.
+
+- **Centered Innovation Defect (`centered_defect`)**:
+  * Added batch-mean centering for gradient innovation tracking (`centered_defect=False` by default):
+
+    $\displaystyle \mu_d = \frac{1}{N}\sum _ {i=1}^N (g _ {t,i} - m _ {t-1,i})$
+
+    $\displaystyle d_t^{(\text{B1})} = (g_t - m _ {t-1}) - \mu_d$
+
+    $\displaystyle d_t^{(\text{B2})} = (g_t - m_t) - \beta_1 \mu_d$
+
+  * In 1.3.3, uncentered innovation $g_t - m_t$ was used directly. On layers where gradients have a non-zero mean (such as normalization scales, biases, and token embeddings), the raw difference caused systematic drift in the second-moment accumulator. Centering eliminates this mean shift.
+
+- **Subnormal-Safe IEEE-754 FP16 Stochastic Rounding**:
+  * Added `f32_to_f16_bits_rne` to handle IEEE-754 subnormal numbers ($|x| < 2^{-14}$) during FP16 stochastic rounding.
+  * In 1.3.3, stochastic rounding added 13-bit random noise and masked with `& -8192`. When a floating-point value entered the subnormal range, bitwise addition directly corrupted the exponent field (bit 10), causing values to flip between normal and subnormal representations or trigger hardware flush-to-zero traps.
+  * The updated routine checks for zero exponent, shifts the significand into the subnormal format, and applies round-to-nearest-even tie-breaking before bit truncation.
+
+- **Multi-Backend Execution Engine**:
+  * Replaced the Python-only runtime with a multi-backend execution layer:
+    * **Rust SIMD (`luminav-simd`)**: AVX-512 (512-bit registers), AVX2 with FMA, portable scalar fallback, and Rayon parallel chunk scheduling (`ELEM_CHUNK_SIZE = 2,097,152`).
+    * **Vulkan UMA (`luminav-vulkan`)**: SPIR-V compute pipeline for cross-vendor GPUs (AMD, Intel, Apple Silicon via MoltenVK, and NVIDIA) with zero-copy host-visible memory buffers.
+    * **Triton AOT (`luminav-triton-aot`)**: C-ABI wrapper dispatching ahead-of-time precompiled kernels for 84 GPU architectures (Blackwell SM100/SM120, Hopper SM90, Ada SM89, Ampere SM80/SM86, Turing SM75, and AMD ROCm gfx90a/gfx942/gfx1100).
+    * **Standalone Runtime (`luminamoon.optimizer.standalone`)**: Pure NumPy and C-ABI execution that runs without requiring PyTorch.
+
+### Changed
+- **Parameter `alpha_ss`**: Removed. In 1.3.3, `alpha_ss=0.5` was used for soft-sign scaling $z / (1 + \alpha _ {\text{ss}} |z|)$ in the legacy single-buffer path. In 1.5.3, TD-CLM envelope normalization bounds updates directly to $(-1, 1)$, making soft-sign obsolete.
+- **Parameter `lam`**: Added with default `0.3`. Governs the defect penalty weight in TD-CLM.
+- **Parameter `centered_defect`**: Added with default `False`. Enables batch-mean centering of the innovation defect.
+- **Parameter `bound_type`**: Default changed from `"radial"` (1.3.3) to `"coordinate"` (1.5.3). Coordinate-wise hyperbolic bounding $(B_i \tanh(\Delta \theta_i / B_i))$ evaluates locally on each parameter without computing a tensor-wide norm reduction.
+- **Parameter `bound_ratio`**: Default changed from `0.03` to `0.05` to provide better update headroom on large model architectures.
+- **Parameter `ema_decay`**: Default changed from static `0.8` to `"auto"`. Resolves to `0.80` for Dual-Buffer mode (`buffer=2`) and `0.95` for Single-Buffer mode (`buffer=1`).
+- **Scratch Buffer**: Expanded from 5 floats `[mask_sum, u_sq_sum, p_sq_sum, cautious_scale, bound_scale]` to 6 floats, adding `diff_sum` at index 5 to accumulate batch-mean defect statistics.
+- **Cautious Scaling Floor**: Enforced `cautious_clamp_min=0.5` across all backends to cap maximum cautious energy scaling at $2.0\times$ ($2^1$).
+
+### Fixed
+- **Hardware Mantissa Divergence**: Fixed bitwise output differences across GPU families by replacing hardware SFU `tl.math.tanh` with Padé [13/6] rational approximation.
+- **FP16 Subnormal Bit Corruption**: Fixed exponent corruption in subnormal FP16 stochastic rounding by separating exponent extraction from significand shift logic.
+- **Single-Buffer Kernel Launch Latency**: Reduced single-buffer kernel dispatches from 3 passes to 1 pass, eliminating host-side synchronization bubbles and intermediate L2 atomic reduction overhead.
+- **Non-Zero Mean Gradient Drift**: Resolved directional accumulator drift on bias and normalization layers via `centered_defect`.
+
+### Parameter Comparison (1.3.3 vs 1.5.3)
+
+| Parameter | Version 1.3.3 | Version 1.5.3 | Technical Notes |
+| :--- | :--- | :--- | :--- |
+| `params` | `params` | `params` | Iterable of parameters or parameter groups. |
+| `lr` | `8e-4` | `8e-4` | Base learning rate. |
+| `betas` | `(0.9, 0.999)` | `(0.9, 0.999)` | First and second moment exponential decay rates. |
+| `eps` | `1e-8` | `1e-8` | Numerical stability floor. |
+| `weight_decay` | `8e-2` | `8e-2` | Decoupled weight decay rate. |
+| `tau` | `0.8` | `0.8` | Noise dispersion scale factor. |
+| `alpha_ss` | `0.5` | *(removed)* | Removed; superseded by TD-CLM envelope normalization. |
+| `lam` | *(none)* | `0.3` | Defect penalty weight in TD-CLM (calibrated to $\lambda^* \approx 0.3008$). |
+| `cautious` | `True` | `True` | Directional verification gate ($u_i g_i > 0$). |
+| `cautious_clamp_min` | `0.5` | `0.5` | Maximum cautious scale cap ($2.0\times$). |
+| `buffer` | `2` | `2` | `1` = Single-Buffer (TD-CLM), `2` = Dual-Buffer (CLM). |
+| `centered_defect` | *(none)* | `False` | Enables subtraction of batch-mean defect $\mu_d$. |
+| `stochastic_rounding` | `True` | `True` | Bitwise stochastic rounding on FP16/BF16 weights. |
+| `bound` | `True` | `True` | Enables update trust-region bounding. |
+| `bound_type` | `"radial"` | `"coordinate"` | Default changed to coordinate-wise bounding (zero reduction overhead). |
+| `bound_ratio` | `0.03` | `0.05` | Maximum step magnitude relative to parameter magnitude. |
+| `master_weights` | `False` | `"none"` | `"none"`, `"semi"`, or `"full"`. |
+| `execution` | `"auto"` | `"auto"` | Dynamic routing: SIMD, Vulkan, Triton, or Foreach. |
+| `fused_single_pass` | `False` | `True` | Enables single-pass streaming with EMA scaling. |
+| `ema_decay` | `0.8` | `"auto"` | Automatically sets `0.80` for dual-buffer, `0.95` for single-buffer. |
+| `fsdp_sync` | `False` | `False` | Cross-rank collective all-reduce synchronization. |
+| `seed` | `1337` | `1337` | PRNG seed for deterministic stochastic rounding. |
+
+---
+
+## [1.3.3] - 2026-10-06
+
+### Fixed
+* **Triton AST Compiler Early-Return Inlining Error:**
+  * Resolved fatal MLIR/CFG code generation failure (`at 35:4: _store_state_buffer(...) ^`) triggered by mid-branch `return` statements in `@triton.jit` helper functions (`_store_param` and `_store_state_buffer`).
+  * Refactored helper control flow into structured, single-exit `if ... elif ... else` branches, ensuring compliant JIT function inlining across all kernel entry points.
+* **Bitcast Width Mismatch (Int64 Promotion in Stochastic Rounding):**
+  * Fixed implicit 64-bit integer promotion caused by Python integer literals (`0x9E3779B9`, `~0x1FFF`, `~0xFFFF`) inside Triton bitwise operations.
+  * Enforced strict 32-bit signed arithmetic via two's complement constants (`-1640531527`, `-2048173461`, `-1028477387`) with explicit `.to(tl.int32)` coercion, and standardized bitwise masks (`-65536` for BF16, `-8192` for FP16), ensuring type-safe 32-bit bitcasting (`tl.int32` to `tl.float32`).
+* **JIT Kernel Call Keyword Arguments:**
+  * Replaced internal keyword argument invocations (`use_sr=False`) with standard positional boolean parameters inside Triton JIT kernels to eliminate compiler argument binding warnings across Triton runtime versions.
+
+### Improved
+* **Throttled Fallback Telemetry & Console De-Noising:**
+  * Added `self._warned_triton_failure` state tracking to prevent multi-line compiler AST dumps from flooding the console across every parameter tensor in a parameter group.
+  * Formatted fallback notifications into a single concise one-line `WARNING` log on initial encounter, routing full tracebacks to `DEBUG` level.
+
+### Verified
+* **End-to-End JIT Execution on NVIDIA Architectures (Tesla T4):**
+  * Validated JIT execution across both Dual-Pass (`fused_single_pass=False`) and Fused Single-Pass (`fused_single_pass=True`) modes in native FP16.
+  * Confirmed non-zero weight update dynamics ($|\Delta W| > 0$) without gradient underflow or weight freezing in pure master-free training for Version 1.3.3.
+
+---
+
+## [1.3.2] - 2026-10-06
+
+### Added
+- **Distributed FSDP Collective Synchronization (`fsdp_sync`)**:
+  - Introduced the `fsdp_sync: bool = False` configuration flag on optimizer initialization, backed by automated runtime guarding via `torch.distributed.is_initialized()`.
+  - Integrated cross-GPU collective communications (`torch.distributed.all_reduce(..., op=torch.distributed.ReduceOp.SUM)`) for global reduction metrics (`mask_sum`, `u_sq_sum`, and `p_sq_sum`) across Triton kernels, single-step execution, and vectorized foreach loops.
+  - Maintains bit-exact scalar consistency for `m_bar`, `cautious_scale`, and `bound_scale` across distributed parameter shards during large-scale pretraining under PyTorch FSDP (Fully Sharded Data Parallel) and DeepSpeed ZeRO-3.
+- **Dedicated Low-Precision Stochastic Quantization Routine (`_sr_quantize`)**:
+  - Factored out bitwise stochastic rounding into a dedicated, modular helper function: `_sr_quantize(target, val_f32, use_sr, generator)`.
+  - Enabled native stochastic rounding on half-precision momentum states (`exp_avg` in BF16/FP16) across all PyTorch CPU/CUDA fallback paths (`_single_step_v2`, `_foreach_step_v2`, `_single_step_v1`, and `_foreach_step_v1`), systematically eliminating gradient stagnation induced by fractional bit truncation (LSB underflow).
+- **Persistent Contiguous Buffer Caching**:
+  - Introduced persistent state cache tensors within `self.state[p]` (`p_contig`, `exp_avg_contig`, `exp_avg_sq_contig`, and `mp_contig`) to stage strided or sliced parameters before kernel launch.
+  - Eliminates dynamic memory allocations (`.contiguous()`) during iterative training loops, preserving PyTorch VRAM caching allocator stability and preventing CUDA Graph capture invalidation.
+- **Zero-Copy Direct Tensor Aliasing for FP32 Master Weights**:
+  - When parameter tensors are natively `torch.float32`, `state["master_param"]` now directly aliases the parameter tensor pointer (`state["master_param"] = p`), avoiding additional master weight VRAM overhead for FP32 layers (0 bytes auxiliary memory footprint).
+- **Checkpoint Sanitizer via State Dict Filter (`state_dict`)**:
+  - Overrode `state_dict()` to automatically purge temporary contiguous cache buffers (`p_contig`, `exp_avg_contig`, `exp_avg_sq_contig`, `mp_contig`), ensuring on-disk checkpoint payloads remain clean, compact, and fully backward-compatible.
+- **Defensive Device Pipeline Flush on Kernel Fault (`torch.cuda.synchronize`)**:
+  - Embedded explicit hardware device synchronization (`torch.cuda.synchronize(p.device)`) within Triton exception recovery handlers prior to PyTorch fallback execution, preventing asynchronous CUDA error cascades into subsequent tensor operations.
+
+### Changed
+- **High-Entropy Bit-Avalanche PRNG Hash Function (`_prng`)**:
+  - Replaced the standard Linear Congruential Generator (LCG) with a MurmurHash3 / SplitMix32 integer bit-avalanche hashing routine:
+    ```python
+    z = idx + seed + 0x9E3779B9
+    z = (z ^ (z >> 16)) * 0x85EBCA6B
+    z = (z ^ (z >> 13)) * 0xC2B2AE35
+    z = z ^ (z >> 16)
+    ```
+  - Delivers uniform pseudo-random noise distributions with zero low-order bit periodicity, significantly refining stochastic rounding fidelity on BF16/FP16 parameters.
+- **Memory-Contiguous Scratch Buffer Realignment for v1 Triton Kernels**:
+  - Re-indexed reduction buffer offsets in Triton v1 kernels:
+    - Pass 1 metrics (`[rms_sum, p_sq_sum]`) mapped to contiguous indices `[0, 1]`.
+    - Pass 2 metrics (`[mask_sum, u_sq_sum]`) mapped to contiguous indices `[2, 3]`.
+  - Enables vectorized contiguous slice all-reduces (`scratch_param[:2]` and `scratch_param[2:4]`) under FSDP without requiring non-contiguous memory gather/scatter overhead.
+- **Multi-GPU Device Index Resolution in RNG Dispatcher (`_get_generator`)**:
+  - Fixed device index resolution when `device` is passed as a generic `torch.device("cuda")` without an explicit index by defaulting to `torch.cuda.current_device()`, avoiding cross-GPU generator key collisions during multi-GPU distributed runs.
+- **Conditional Atomic Reduction Pruning in v2 Pass 1 Kernel**:
+  - Guarded `mask_sum` atomic accumulation in `_lumina_v2_pass1_kernel` with `if cautious:`, bypassing CTA reductions and L2 atomic additions when `cautious=False`.
+- **Expanded Master Weight Precision Configuration Aliases**:
+  - Added the `"half"` alias for `"semi"` mode alongside `"fp32"` and `"masterweight"` aliases for `"full"` mode.
+- **Metadata & Documentation Header Update**:
+  - Bumped specification version to `1.3.2`.
+
+### Fixed
+- **Singular Zero-Division & Step-Zero Bias Correction (`safe_base_step`)**:
+  - Introduced `safe_base_step = max(base_step, 1)` and `bc2_base = max(1.0 - (beta2 ** safe_base_step), 1e-15)` across Triton, single-step, and foreach loops.
+  - Fully resolves potential `ZeroDivisionError` and NaN blowups in `c1` / `c2` computations when steps evaluate at `step == 0`.
+- **Numerical Overflow Clamp in Sigmoid-Based Tanh Fallback**:
+  - Applied register clamp bounds `x_clamped = tl.maximum(tl.minimum(x, 20.0), -20.0)` in analytical fallback `_triton_tanh_fast`, preventing `tl.exp(-2.0 * x)` from overflowing to `inf` or `nan` on devices lacking hardware SFU tanh instructions.
+- **Missing Argument Forwarding in PyTorch Single-Step v1**:
+  - Corrected `_single_step_v1` method signature and call-sites to accept and propagate `scratches`, eliminating runtime `TypeError` during fallback invocation.
+- **Multi-Rank Synchronization Skew in Sharded Parameter Bounding**:
+  - Aligned tensor norms and radial bounding factors across sharded ranks via synchronized global scalar reductions, preventing parameter update drift across distributed ranks.
+
+### Performance & Engine Refactoring
+- **Zero Dynamic VRAM Allocation on Strided / Sliced Tensors**:
+  - Persistent pre-allocated contiguous staging buffers avoid PyTorch caching allocator churn and eliminate CUDA Graph memory address invalidations.
+  - Preserves deterministic memory footprints across long pre-training runs.
+- **Optimized Distributed Collective Bandwidth**:
+  - Grouping scalar reductions into contiguous vector slices compresses distributed synchronization overhead into a single collective kernel launch per pass.
+
+---
+
+## [1.3.0] - 2026-09-23
+
+### Added
+- **Fused Single-Pass Kernel with Post-Step EMA (`fused_single_pass`, `ema_decay`)**:
+  - Introduced `_lumina_v2_fused_single_pass_kernel` coupled with an asynchronous CTA micro-kernel `_lumina_v2_post_step_ema_kernel[(1,)]`.
+  - When `fused_single_pass=True`, avoids two-pass global reductions by leveraging running Exponential Moving Average (EMA) estimates of `cautious_scale` ($1 / \bar{M}$) and `bound_scale` ($\tanh(r) / r$).
+  - Unifies gradient filtering, momentum accumulation, variance dispersion, cautious masking, radial bounding, decoupled weight decay, stochastic rounding, and block partial reductions into a **single unified VRAM-resident pass**, reducing global GPU memory bandwidth traffic per optimizer step.
+- **Hybrid Master Weight Precision Architecture (`master_weights`)**:
+  - Added multi-tier master weight flexibility via `master_weights: Union[bool, str] = False`:
+    - `"none"` / `False` *(Default)*: Pure master-free low-precision training leveraging on-chip bitwise stochastic rounding directly on native half-precision weights.
+    - `"semi"` / `"hybrid"` / `"half"`: 32-bit FP32 master parameters (`state["master_param"]`) coupled with 16-bit low-memory optimizer states (`exp_avg`, `exp_avg_sq` in BF16/FP16), saving ~50% optimizer state VRAM compared to traditional FP32 setups while maintaining full FP32 weight accumulation accuracy.
+    - `"full"` / `"fp32"` / `"masterweight"`: Traditional full FP32 master parameters with full FP32 optimizer states for standard baseline comparisons.
+- **In-Kernel In-Flight NaN / Inf Gradient Sanitization**:
+  - Embedded register-level defensive sanitization across all Triton GPU kernels: `is_finite_g = (g == g) & (g < 1e38) & (g > -1e38); g = tl.where(is_finite_g, g, 0.0)`.
+  - Replaces non-finite gradients and subnormal corruptions with zero in registers before they can contaminate momentum (`exp_avg`) or variance (`exp_avg_sq`) state buffers.
+- **Low-Precision State Buffer Serialization (`_store_state_buffer`)**:
+  - Added a dedicated Triton state serialization routine supporting FP32, BF16, and FP16 state representations with optional bitwise stochastic rounding applied directly to momentum buffers.
+- **Multi-Device & Cross-Accelerator Hardware Context Isolation (`_get_device_context`)**:
+  - Unified device stream scoping supporting NVIDIA CUDA (`torch.cuda`), Intel XPU (`torch.xpu`), and modern PyTorch 2.x unified accelerator interfaces (`torch.accelerator.device`), eliminating multi-GPU stream hijacking and cross-device race conditions.
+- **Strict Determinism Auto-Routing**:
+  - Automatically queries `torch.are_deterministic_algorithms_enabled()`. When enabled under `execution="auto"`, dynamically routes execution to C++ vectorized `foreach` to preserve bitwise cross-run reproducibility, circumventing non-deterministic floating-point hardware atomic reordering.
+- **Granular Fault Recovery with State Guarding (`completed_indices`)**:
+  - Implemented per-parameter completion tracking in `_dispatch_triton`. If a kernel encounters an unexpected runtime fault, completed parameters are preserved while uncompleted ones fall back cleanly to `foreach`.
+  - Added `_single_step_v2_apply_param` and `_single_step_v1_apply_param` to safely finalize parameter updates if Pass 1 had already committed state transitions, strictly preventing double-update state corruption.
+
+### Changed
+- **Per-Parameter Persistent Scratch Tensors (Elimination of Global Slicing)**:
+  - Replaced the global dynamic slice allocator (`self._scratch_tensors[dev][:count]`) with isolated per-parameter persistent state allocations (`state["scratch"] = torch.zeros((5,), device=p.device, dtype=torch.float32)`).
+  - Memory pointers for `[mask_sum, u_sq_sum, p_sq_sum, cautious_scale, bound_scale]` remain completely invariant throughout the entire training lifecycle, preserving pointer stability for **CUDA Graph Capture** (`torch.accelerator.Graph`) and TorchInductor AOT compilation without memory aliasing.
+- **Analytically Exact Nesterov Bias Correction ($\beta_1^{\text{step} + 1}$)**:
+  - Corrected first-moment bias correction from the classical Adam formulation ($1.0 - \beta_1^{\text{step}}$) to the exact analytical expectation of the Nesterov blended momentum: `bc1_nes = 1.0 - beta1^(step + 1)`. Removes residual momentum bias during early warmup iterations.
+- **Adaptive CTA Tile Sizing and Dynamic Thread Warp Scaling**:
+  - Introduced hardware-aware CTA tile scaling based on parameter volume: parameters with $\ge 2,097,152$ elements (2M+) dynamically configure `BLOCK_SIZE = 2048` and `num_warps = 8` (versus `BLOCK_SIZE = 1024`, `num_warps = 4` for smaller tensors).
+  - Cuts total thread-block grid counts and L2 cache atomic addition pressure by 50% on large MLP, Attention Projection, and Vocabulary layers.
+- **Hardware-Native Tanh Instruction Dispatch (`tl.math.tanh` / `tl.tanh`)**:
+  - Upgraded `_triton_tanh_fast` to dynamically inspect compiler capabilities, binding directly to native PTX Special Function Unit (SFU) instructions (`tanh.approx.f32`) on modern Triton compilers while retaining analytical sigmoid fallbacks for legacy environments.
+- **PRNG Decorrelation between Momentum and Parameter Updates**:
+  - Separated RNG seed streams using bitwise alternating mask shifts: `seed_p = (seed + 0x55555555) & 0x7FFFFFFF`, mathematically eliminating cross-coupling noise correlation between state buffer rounding and parameter update rounding.
+- **Bumped specification version metadata to 1.3.0** across `luminav.py`.
+
+### Fixed
+- **Floating-Point Cancellation in Radial Bounding ($r \to 0$)**:
+  - Fixed precision jitter and roundoff degradation near zero by adding an asymptotic Taylor-series guard across Triton and PyTorch bounding engines: `torch.where(r < 1e-4, 1.0, tanh(r) / r)`.
+- **Artificial Denominator Dilation in Dispersion Scale ($\sigma$)**:
+  - Replaced additive epsilon inflation (`sigma + 1e-6`) with a non-distorting floor clamp (`tl.maximum(sigma, 1e-12)`), preventing artificial dampening of subtle gradient signals.
+- **Singular Zero-Division in Parameter Norm Clamping**:
+  - Replaced `norm_p + 1e-6` with `torch.clamp_min(norm_p, 1.0)` / `tl.maximum(norm_p, 1.0)`, maintaining numerical stability even when parameters are initialized near zero.
+- **Transient Memory Pool Spikes during PyTorch Fallback Stochastic Rounding**:
+  - Refactored `_sr_update` to stream stochastic noise generation in bounded chunks of 1,048,576 elements (~4 MB chunk size), eliminating multi-hundred megabyte temporary int32 tensor spikes during fallback execution on large parameter matrices.
+- **Non-Contiguous Buffer Safety & Strided Memory Copy**:
+  - Added strict contiguous layout verification for parameters, states, and master weights with in-place synchronization (`p.copy_(p_contig)`), guarded by a one-time developer warning.
+
+### Performance & Engine Refactoring
+- **~50% Global Memory Bandwidth Reduction via Fused Single-Pass Mode**:
+  - By unifying parameter loading, gradient evaluation, state transformation, and bounding into one GPU pass under `fused_single_pass=True`, VRAM read/write cycles are halved for cautious and radially bounded steps.
+- **Sub-Microsecond L2 Hardware Atomic Reduction Architecture**:
+  - Formally documented the hybrid Intra-Block Tree Reduction + Inter-Block Hardware Atomic pattern. With `BLOCK_SIZE=2048`, a 10M-parameter tensor issues only 4,883 atomic additions directly into GPU L2 cache crossbars, executing in < 0.5 µs and strictly outperforming multi-tier cascaded reduction micro-kernels by eliminating 3–5 µs driver dispatch bubbles.
+- **Static Buffer Address Invariance for CUDA Graphs & TorchInductor AOT**:
+  - Persistent 5-slot state scratch buffers eliminate all dynamic slicing and memory reallocations during optimization loops, preserving pointer address invariance for CUDA Graph replay.
+
+---
+
+## [1.2.1] - 2026-09-17
+
+### Added
+- **In-Kernel GPU Parameter Norm Reduction (`p_sq_sum_ptr`)**: Fused parameter Euclidean norm squared $\sum(p_i^2)$ calculation directly into Pass 1 Triton reduction kernels (`_lumina_v2_pass1_kernel` and `_lumina_v1_pass1_kernel`). Pass 2 and Pass 3 now load and resolve `norm_p` (`tl.sqrt(tl.maximum(p_sq_sum, 0.0))`) entirely inside GPU SRAM/registers without host intervention.
+- **Stateless CPU Integer Multiplicative PRNG Hashing**: Replaced dynamic GPU RNG tensor allocation with a pure CPU integer hash using Knuth's 32-bit golden ratio constant: `(step * 0x9E3779B9 + i * 10007) & 0x7FFFFFFF`. Generates uncorrelated 31-bit integer seeds per layer with zero GPU kernel launch overhead and zero PCIe host-device latency.
+- **Isolated Generator Cache for Fallback Engine (`self._generators`)**: Added an internal per-device `torch.Generator` cache, ensuring stochastic rounding in `_sr_update` operates in complete isolation from PyTorch's default CUDA generator state.
+
+### Changed
+- **Consolidated Scratch Buffer Zeroing**: Removed redundant per-parameter slice zeroing inside parameter loops. Buffer zeroing is now batched into a single consolidated `scratch.zero_()` call per step inside `_get_scratch_buffer`.
+- **Scratch Buffer Stride Expansion**:
+  - `buffer=2`: Expanded stride from 2×N to 3×N elements per parameter to host `[mask_sum, u_sq_sum, p_sq_sum]`.
+  - `buffer=1`: Expanded stride from 3×N to 4×N elements per parameter to host `[rms_sum, mask_sum, u_sq_sum, p_sq_sum]`.
+- **Bumped specification version metadata to 1.2.1**.
+
+### Fixed
+- **Host-Device Synchronization & Pipeline Stalls (`.item()` Bottleneck)**: Removed `norm_p = float(p_contig.float().norm().item())` and `seed = torch.randint(...).item()` from Triton step pipelines. Eliminated 200–400 synchronous PCIe roundtrips and GPU pipeline stalls per step on deep architectures (e.g., 28-layer Transformers), restoring full asynchronous GPU compute utilization.
+- **CUDA Graphs & `torch.compile` Graph Breaks**: Fixed fatal graph capture crashes and TorchDynamo graph breaks caused by blocking host-device synchronization during radial bounding and seed generation. The optimizer step is now compatible with CUDA Graphs (`torch.accelerator.Graph`) and TorchInductor compilation.
+- **Transient VRAM Allocator Thrashing**: Eliminated repeated temporary FP32 parameter cloning (`p_contig.float()`) during norm evaluation, preventing memory pool fragmentation in tight VRAM regimes.
+- **Global RNG Pollution & Determinism Compliance**: Fixed global RNG state mutation in fallback stochastic rounding. Solved reproducibility drift in upstream Dropout, DataLoader shuffling, and RLHF reward modeling, ensuring strict adherence to PyTorch 2.10+ `torch.use_deterministic_algorithms(True)`.
+
+### Performance & Engine Refactoring
+- **Batch Kernel Launch Overhead Elimination**: Removed up to 400–800 individual `cudaMemsetAsync` micro-calls per step previously triggered by per-slice `.zero_()` invocations on 1-element scratch tensors.
+- **Single-Pass Bypass for Coordinate Bounding**: Streamlined `_triton_step_v2` to immediately bypass two-pass reductions and invoke `_lumina_v2_single_pass_kernel` when `cautious=False` and coordinate bounding (`bound_mode == 2`) is selected.
+
+---
+
+## [1.2.0] - 2026-09-17
+
+### Added
+- **Automatic Dynamic FP16 Cliff Governor:** Built-in asymptotic boundary governor (`_triton_tanh_fast` in GPU register and PyTorch fallback) that automatically activates when `p.dtype == torch.float16`. Prevents unclipped steps from pushing parameters, activations, and residual streams into the IEEE-754 FP16 overflow cliff (> 65,504). Remains completely dormant (zero overhead) on `bfloat16` (TPU / Ampere+) and `float32`.
+- **Smooth Asymptotic Parameter Bounding (`bound`, `bound_type`, `bound_ratio`):** Added non-intrusive step bounding with two mathematically continuous formulations:
+  - `"radial"`: Direction-Preserving Radial Squashing (`update * tanh(r) / r`) preserving gradient direction (Cosine Similarity = 1.000).
+  - `"coordinate"`: Dual-Hyperbolic Coordinate Squashing for fine-grained per-element damping.
+  - Configurable via constructor: `bound: bool = True`, `bound_type: str = "radial"`, `bound_ratio: float = 0.03`.
+
+### Changed
+- **Increased Default `cautious_clamp_min` from `0.2` to `0.5`:** Enforces an exact power-of-two maximum energy scaling factor (2.00x, $2^1$). Eliminates floating-point mantissa roundoff distortion in IEEE-754 hardware.
+- Bumped specification version metadata to `1.2.0`.
+
+### Fixed
+* **FP16 Step Divergence & NaN Overflow at High Learning Rates:** Solved step divergence and subsequent NaN crashes when training deep 28-layer Transformer architectures (e.g., Qwen3-0.6B) in pure FP16 at nominal learning rates (`lr = 8e-4`).
+* **Sparse Embedding & LM-Head Amplification:** Mitigated 5x over-amplification on sparse vocabulary gradients where active token counts are much smaller than total tensor parameters ($N \gg N_{\text{active}}$).
+* **Verified stable 1,000-step pretraining from scratch on Qwen3-0.6B** (seq length 512, pure FP16, `lr = 8e-4`) on Nvidia Tesla T4 with zero gradient clipping and zero LR scheduler (loss dropped from 12.09 down to ~5.7 with zero NaNs).
+* **Weight Decay Numerical Parity on Fallback Engine:** Resolved a silent precision-stagnation issue in `_single_step` and `_foreach_step`. Previously, decoupled weight decay (`p.mul_(1.0 - lr * weight_decay)`) was executed directly on native FP16/BF16 tensors prior to `_sr_update`. Due to IEEE-754 Round-to-Nearest (RTN) limits, `(1 - lr * weight_decay) ≈ 0.999936` rounded back to `1.0`, causing weight decay to freeze on half-precision parameters. Weight decay is now unified into FP32 registers inside `_sr_update` alongside stochastic rounding, matching the numerical behavior with the Triton kernel.
+* **Fallback Foreach Indentation & Scope Alignment:** Corrected tensor iteration scope and indentation for `torch._foreach_add_` in non-SR fallback branches.
+* **In-Place GPU Clamping in `_apply_bound`:** Replaced dynamic GPU scalar allocations (`torch.tensor(1.5e-4, device=p.device)`) with native `torch.clamp_min`, eliminating transient CUDA malloc spikes per update step.
+
+### Performance & Engine Refactoring
+* **Unified Bound Execution Pipeline:** Added an immediate short-circuit guard (`if lr == 0.0: return update`) and streamlined the execution flow: intermediate updates are scaled in FP32 (`step_val = lr * u_float`), passed through radial/coordinate bounding, chained cleanly into the FP16 Cliff Governor, and returned in a single normalized step.
+
+---
+
+## [1.1.5] - 2026-09-15
+
+### Added
+- **Native Hardware Capability Detection (`_supports_native_bf16`):** Added automatic device capability detection via `torch.cuda.is_bf16_supported()` to identify Ampere SM80+ architectures and streamline native execution paths.
+- **Hugging Face Repository Reference:** Added official Hugging Face repository link in module docstrings.
+
+### Changed
+- **Dynamic Epsilon Clamping for FP16:** Dynamically floors effective epsilon to `max(eps, 1e-4)` when parameters or states are in `torch.float16`. Prevents bias-corrected scale factor `c2` from underflowing into IEEE-754 subnormal/zero limits on step 1.
+- Bumped specification version to `1.1.5`.
+
+### Fixed
+- **FP16 Step-1 `NaN` from Epsilon Underflow:** Fixed `0.0 / 0.0 = NaN` division occurring on initial training steps when parameters receive zero gradients (e.g., unselected vocabulary embeddings, padding tokens, or dropout paths).
+- **Second-Moment Variance Saturation (`+inf`):** Added safe upper-bound register clamping (`tl.minimum(v_new, 65000.0)` in Triton kernels and `.clamp_(0.0, 65000.0)` in PyTorch C++ foreach/single loops). Prevents unclipped initial pretraining gradient surges ($|g| > 256$) from overflowing the FP16 maximum dynamic range (65,504) into irreversible `+inf`.
+- **Zero-Sigma Division Guard:** Injected explicit `sigma + 1e-6` denominator guards across all Triton kernel passes and PyTorch update steps to ensure numerical safety during cold-start iterations.
+- Verified stable 100-step pretraining from scratch on `SmolLM-135M` using Wikimedia Wikipedia in pure FP16 on Nvidia Tesla T4 GPU with zero gradient clipping.
+
+---
+
+## [1.1.0] - 2026-09-11
+
+### Added
+- **Bitwise Stochastic Rounding (SR):** Integrated on-chip PRNG integer bit-manipulation hash in Triton kernels and vectorized PyTorch fallback for native FP16/BF16 training without master weights.
+- Unified Triton store routine introducing compile-time branching (`dtype_mode` / `use_sr: tl.constexpr`) with explicit safe casting for FP16 and BF16 pointers.
+- Vectorized fallback improvements utilizing C++ multi-tensor operations when stochastic rounding is disabled or tensors are FP32.
+- State buffer contiguity validation (`exp_avg_contig`, `exp_avg_sq_contig`) ensuring safe linear memory addressing before kernel dispatch.
+- Declarative configuration file (`config.json`) defining hyperparameters and optimizer presets.
+
+### Changed
+- Refactored PRNG integer arithmetic to strictly stay within signed 32-bit integer limits ($< 2^{31} - 1$), eliminating JIT compilation failures on newer Triton releases.
+- State buffers (`exp_avg`, `exp_avg_sq`) now initialize with strict contiguous memory format.
+
+### Fixed
+- Fixed `ValueError: Scalar out of range for type int32` and Triton LLVM compile errors during unsigned 32-bit bitmask operations.
+- Resolved kernel fallback hangs and compilation overhead on CUDA devices.
+
+---
+
+## [1.0.2] - 2026-09-08
+
+### Added
+- Official Apache-2.0 license notice and copyright headers.
+- Standardized package exports (`__all__`, `__version__`, `__author__`, `__license__`).
+
+---
+
+## [1.0.1] - 2026-08-31
+
+### Fixed
+- **Memory Synchronization Fix (`423d171`):** Fixed a critical bug where non-contiguous parameter tensors (e.g., transposed linear layers or sliced weights) were updated on temporary contiguous copies without reflecting changes back to model weights. Added `p.copy_(p_contig)` after Triton execution.
+
+---
+
+## [1.0.0] - 2026-08-30
+
+### Added
+- Initial release of the LuminaV optimizer.
+- **Zero Master-Weight Copies:** Direct parameter updates in native half-precision without allocating 4-byte FP32 master weights.
+- **Hyperbolic Tangent (`tanh`) Bounding Envelope:** Projects normalized momentum ratio into `(-1.0, 1.0)` to eliminate exploding gradient shocks.
+- **Directional Alignment Gate:** Cautious update masking that zeroes out momentum steps opposing mini-batch gradient directions.
+- **Central Innovation Variance:** Noise dispersion tracking $(g_t - m_t)^2$ decoupling mini-batch turbulence from directional descent.
+- Dual-buffer mode (`buffer=2`, LuminaV-2B) and single-buffer mode (`buffer=1`, LuminaV-1).
+- Custom fused OpenAI Triton kernels with automated fallback execution.

changelogs/README.md

@@ -1,25 +1,25 @@
-# LuminaMoon Changelogs
-
-This directory separates release tracking into two distinct lineages to eliminate version ambiguity between the software distribution and the underlying mathematical specification:
-
-| Document | Current Version | Scope |
-| :--- | :--- | :--- |
-| [**Software Changelog**](SOFTWARE_CHANGELOG.md) | **`v1.1.0`** | Tracks the `luminamoon` package distributed on PyPI and GitHub (`pip install luminamoon`), covering the Rust workspace crates, CPython FFI bindings, Data Engine, LSO-Eval, and CI/CD infrastructure. |
-| [**LuminaV Algorithm Changelog**](LUMINAV_ALGORITHM_CHANGELOG.md) | **`v1.6.0`** | Tracks the mathematical formulation, numerical invariants, stochastic rounding algorithms, and GPU/SIMD kernel evolution of the **LuminaV Optimizer** from `v1.0.0` through the `v1.6.0` specification. |
-
-## Version Introspection in Python
-
-```python
-import luminamoon
-from luminamoon import LuminaV
-import luminamoon.data as lmd
-
-# 1. Software Package Version (PyPI / Cargo / Data Engine)
-print(luminamoon.__version__)      # "1.1.0"
-print(lmd.get_version())           # "1.1.0"
-
-# 2. LuminaV Algorithm Math Version
-print(LuminaV.ALGORITHM_MATH)      # "1.6.0"
-print(LuminaV.ALGORITHM_SPEC)      # "1.6.0" (Backward-compatible alias)
-print(LuminaV.getVersion())        # "1.6.0"
-```
+# LuminaMoon Changelogs
+
+This directory separates release tracking into two distinct lineages to eliminate version ambiguity between the software distribution and the underlying mathematical specification:
+
+| Document | Current Version | Scope |
+| :--- | :--- | :--- |
+| [**Software Changelog**](SOFTWARE_CHANGELOG.md) | **`v1.1.0`** | Tracks the `luminamoon` package distributed on PyPI and GitHub (`pip install luminamoon`), covering the Rust workspace crates, CPython FFI bindings, Data Engine, LSO-Eval, and CI/CD infrastructure. |
+| [**LuminaV Algorithm Changelog**](LUMINAV_ALGORITHM_CHANGELOG.md) | **`v1.6.0`** | Tracks the mathematical formulation, numerical invariants, stochastic rounding algorithms, and GPU/SIMD kernel evolution of the **LuminaV Optimizer** from `v1.0.0` through the `v1.6.0` specification. |
+
+## Version Introspection in Python
+
+```python
+import luminamoon
+from luminamoon import LuminaV
+import luminamoon.data as lmd
+
+# 1. Software Package Version (PyPI / Cargo / Data Engine)
+print(luminamoon.__version__)      # "1.1.0"
+print(lmd.get_version())           # "1.1.0"
+
+# 2. LuminaV Algorithm Math Version
+print(LuminaV.ALGORITHM_MATH)      # "1.6.0"
+print(LuminaV.ALGORITHM_SPEC)      # "1.6.0" (Backward-compatible alias)
+print(LuminaV.getVersion())        # "1.6.0"
+```

changelogs/SOFTWARE_CHANGELOG.md

@@ -1,261 +1,261 @@
-# LuminaMoon Software Changelog
-
-All notable software, packaging, FFI, and infrastructure changes to the `luminamoon` monorepo and PyPI distribution (`pip install luminamoon`) are documented in this file.
-
-The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
-
-For the mathematical and kernel evolution of the **LuminaV Optimizer** specification (`v1.0.0` to `v1.6.0`), see [`LUMINAV_ALGORITHM_CHANGELOG.md`](LUMINAV_ALGORITHM_CHANGELOG.md).
-
----
-
-## [1.1.0.post3] - 2026-11-07
-
-### Changed
-- **Academic Citation Consolidation**:
-  * Removed redundant `CITATION.cff` metadata file to consolidate citation records directly into Zenodo (DOI: [`10.5281/zenodo.23978801`](https://doi.org/10.5281/zenodo.23978801)) and repository `README.md`.
-- **Data Engine Version Introspection (`python/luminamoon/data/version.py`)**:
-  * Streamlined data version checking so `get_version()` directly returns the active package version string without disk or mutable state side effects.
-  * Maintained full backward compatibility for `list_versions()` and `set_version()` with informative `DeprecationWarning` notices explaining that the data engine operates on the unified native SIMD implementation.
-  * Removed internal cache directory cloning and dynamic module loading routines from `~/.cache/luminamoon/versions`.
-  * Updated `data/README.md` to document direct version inspection via `luminamoon.data.get_version()`.
-- **Documentation & Technical Guides**:
-  * Rewrote technical guides, component documentation, and reference overviews across `docs/` and `data/docs/` in direct, humble, and accessible language for human readability.
-  * Preserved core algorithmic identities including the Triangle Family, Triangle Inequality Envelope, Gradient-Momentum Residual, and Triangle-Defect Penalty without inflated descriptors.
-  * Normalized data engine documentation in `python/luminamoon/data` and `data/` to consistent English terminology.
-  * Aligned the MinHash documentation in `docs/dataloader/sources_and_streaming/README.md` with the Rust implementation using XXH3-64 seed multipliers.
-
-### Refined
-- **Evaluation Specification Artifacts (`eval/`)**:
-  * Updated `eval/lso_eval_specification.tex` with refined layout and mathematical notation.
-  * Added updated compiled specification manuscript `eval/LSO_Eval.pdf` and removed legacy PDF files.
-  * Aligned `tools/sync_huggingface.py` and `eval/README.md` to track `eval/LSO_Eval.pdf`.
-- **PosRaGLU Activation Gating (`luminamoon.nn`)**:
-  * Added explicit positive anchor support ($b > 0$) for `pos_ra_gate` and `PosRaGLU`, preserving unit derivative at the origin while bounding output in $(0, 1)$.
-
-### Fixed
-- **Markdown Parameter Table Pipe Escaping**:
-  * Escaped pipe characters in inline code notation (`\|d_t\| = \|g_t - m_{t-1}\|`) within the README parameter table to prevent GitHub Flavored Markdown (GFM) table parsers from prematurely splitting table cells.
-- **Static Type Analysis (`luminamoon.nn._compat`)**:
-  * Declared `_rs_engine` at module scope with multi-tier import fallbacks to eliminate Pyright unresolved import warnings.
-
-### Infrastructure
-- **Continuous Integration & Release Automation**:
-  * Added Python runtime setup to the Rust test workflow in `.github/workflows/ci.yml` so linkers reliably locate shared Python libraries during PyO3 FFI tests on updated runner images.
-  * Streamlined GitHub Release note generation by eliminating redundant boilerplate headers and prioritizing direct changelog entries.
-
----
-
-## [1.1.0.post2] - 2026-11-05
-
-### Fixed
-- **Rust Arrow & Parquet Dependencies (`RUSTSEC-2026-0442`)**:
-  * Updated `parquet`, `arrow-array`, `arrow-schema`, and `arrow-ipc` to `60.1.0` in workspace manifests (`Cargo.toml` and `Cargo.lock`).
-  * Resolves advisory **RUSTSEC-2026-0442** (CVE-2026-111959) in `arrow-data`, preventing potential out-of-bounds reads when validating RunEndEncoded arrays during dataset ingestion.
-
----
-
-## [1.1.0] - 2026-11-04
-
-### Added
-- **Hardware-Accelerated Triangle Family Fused Triton Kernels (GPU JIT & Rust AOT)**:
-  * **Python JIT (`_lumina_t1_single_pass_kernel`, `_lumina_t2_fused_single_pass_kernel`)**: Added dedicated fused register kernels for the Triangle Family (`buffer="T1"` and `buffer="T2"`), replacing eager PyTorch chunking and eliminating CPU-GPU `.item()` sync stalls, bringing T2 execution latency from 20.7 ms down to native GPU speed (~0.9–1.0 ms).
-  * **Rust Triton AOT Crate (`luminav-triton-aot`)**: Added Ahead-Of-Time compiled kernels for `T1` and `T2` across 12 architectures (Turing, Ampere, Ada, Hopper, Blackwell, CDNA, RDNA3), expanding the embedded manifest to 96 binary artifacts.
-- **Strict Version Capability Enforcement**:
-  * Enforced specification capabilities across historical LuminaV algorithm versions (`1.6.0`, `1.5.5`, `1.5.4`, `1.5.3`): locking to `1.5.4` or `1.5.5` strictly rejects features introduced in later versions (such as `buffer="T2"` or `fuse_tensors=True`), guaranteeing reproducibility for academic benchmarks.
-
-### Added
-- **Ra-Family (Rational, Tanh-Free & Square-Root-Free Neural Architecture)**:
-  * **Zero Mandatory PyTorch Dependency**: The `luminamoon` package remains lightweight (~2.4 MB) with zero required dependencies (`dependencies = []`). The new `luminamoon.nn` module automatically detects the runtime environment (`_compat.py`), inheriting from `torch.nn.Module` when PyTorch is installed, or falling back cleanly to standalone NumPy buffers when PyTorch is absent.
-  * **`RaNorm` (`luminamoon.nn.RaNorm`)**: Token normalization using $L_1$ mean absolute deviation instead of root-mean-square. Eliminates square-root evaluations and amortizes a single scalar reciprocal per token vector.
-  * **`RaBatchNorm2d` (`luminamoon.nn.RaBatchNorm2d`)**: 2D batch normalization using cross-batch mean absolute deviation (MAD). Includes inference weight folding (`fold_into_conv`) that merges running statistics into preceding `Conv2d` layers with zero additional inference latency.
-  * **`RaTanh` & `ra_tanh`**: Rational hyperbolic tangent with non-vanishing algebraic gradient tails $O(1 / |x|^3)$ and unit derivative at origin.
-  * **`RaGELU` & `ra_gelu`**: Rational GELU activation using rational polynomial approximations.
-  * **`PosRaGLU` (`luminamoon.nn.PosRaGLU`)**: Gated Linear Unit with strictly positive rational gating bounded in $(0, 1)$. Eliminates quadratic activation outlier explosion for stable low-precision (INT4/FP8) quantization.
-  * **`RaGLU` & `TriGLU`**: Standard and triple rational gated linear unit variants.
-  * **`RaAttention` & `RaAttn` (`luminamoon.nn.RaAttention`, `luminamoon.nn.RaAttn`)**: Multi-head attention using Cayley-Padé rational softmax. Supports causal masking, QK-Norm (`qk_norm=True`), dropout, and custom head dimension scaling.
-  * **`ra_softmax` (`luminamoon.nn.ra_softmax`)**: Rational softmax with exact compact support for $\Delta \ge 16$, guaranteeing sum of numerators $\ge 1.0$ without requiring an epsilon floor.
-  * **Dual Hardware Acceleration (Triton JIT GPU & Rust AVX-512/AVX2 SIMD CPU)**:
-    - Added Triton GPU kernels (`_ra_norm_fwd_kernel`, `_pos_ra_glu_fwd_kernel`, `_ra_softmax_fwd_kernel`) in `luminamoon.nn.triton_kernels` for register-fused execution without intermediate global memory roundtrips.
-    - Added native Rust SIMD kernels in `luminav-simd` with AVX2/FMA vectorization and PyO3 bindings for high-throughput CPU token normalization and rational softmax.
-- **Academic Aliases (`luminamoon.nn`)**:
-  * Exported standardized academic aliases across neural components: `L1Norm`, `L1LayerNorm`, `L1BatchNorm2d`, `RationalTanh`, `rational_tanh`, `RationalGELU`, `rational_gelu`, `RationalGLU`, `BoundedRationalGLU`, `rational_softmax`, `pade_softmax`, `RationalAttention`, and `PadeAttention`.
-- **Theoretical References & Literature Documentation**:
-  * Added `references/README.md` and `references/bibliography.bib` documenting literature mappings and rational approximations, paired with `docs/references.md` in the documentation.
-- **Empirical Evaluation**:
-  * Updated `eval/LSO-Eval.pdf` with the latest benchmark convergence findings and empirical evaluation report.
-- **Unit Tests**: Added 32 unit tests in `tests/test_nn_ra_family.py` covering mathematical invariants, convolution folding, isolated zero-torch subprocess execution, and hardware introspection.
-
-### Changed
-- **Security Policy**: Streamlined `SECURITY.md` supported version matrix (`luminamoon >= 1.1.0`, `luminav >= 1.6.0`) with simplified reporting guidelines.
-
----
-
-## [1.0.9] - 2026-11-03
-
-### Added
-- **LuminaV v1.6.0 — T-FAMILY (Triangle Family) tanh-free Modes (`buffer="T1"` & `buffer="T2"`)**:
-  * **`buffer="T2"` (2-Buffer Triangle Family Mode)**: Keeps the 2-buffer state (`exp_avg` and `exp_avg_sq`) without using $\tanh$. Computes the gradient-momentum residual $d_t = g_t - m _ {t-1}$ and bounds the lookahead update via the **Triangle Inequality Envelope**:
-
-    $\displaystyle \mathcal{E} _ t^{(\text{T2})} = w_1 |m _ {t-1}| + w_2 |g_t| + 0.5\lambda |d_t| + 0.5\lambda \sigma_t + \epsilon _ {\text{eff}}, \qquad u_t^{(\text{T2})} = \frac{\tilde{m} _ t}{\mathcal{E} _ t^{(\text{T2})}} \in (-1, 1)$
-
-    By the Triangle Inequality, $|w_1 m _ {t-1} + w_2 g_t| \le w_1 |m _ {t-1}| + w_2 |g_t| \le \mathcal{E} _ t^{(\text{T2})}$, which guarantees the update step $u_t^{(\text{T2})}$ strictly remains within $(-1, 1)$ without evaluating $\tanh$. The term $0.5\lambda |d_t|$ acts as a **triangle-defect penalty**, while $0.5\lambda \sigma_t$ dampens belief-variance dispersion.
-  * **`buffer="T1"` (1-Buffer Triangle Family Mode)**: Pairs the single-buffer **Triangle-Defect** denominator ($\mathcal{E} _ t^{(\text{T1})} = w_1 |m _ {t-1}| + w_2 |g_t| + \lambda |d_t| + \epsilon _ {\text{eff}}$ with residual $d_t = g_t - m _ {t-1}$) with the rational update bounds below via the Triangle Inequality, eliminating the second-moment accumulator ($v_t$) and reducing optimizer state memory by 50%.
-  * **Rational Update Bounds (for `T1` & `T2`)**: Replaces $\tanh$ and $\sqrt{\cdot}$ in the coordinate, radial, and FP16 update bounds with simple rational formulas across Rust, PyTorch, FSDP, and NumPy:
-
-    $\displaystyle b_i = \mathrm{bound} _ {\text{ratio}} \cdot (|w_i| + 10^{-2}), \qquad \Delta w_i = \frac{b_i (b_i + |s_i|) s_i}{b_i (b_i + |s_i|) + s_i^2}, \qquad \mathrm{scale} _ {\text{radial}} = \frac{1 + r}{1 + r + r^2}$
-- **100% Backward Compatibility & Zero Breaking Changes**:
-  * Existing code and checkpoints using `buffer=2` (default), `buffer=1`, or `version="1.5.5"` continue to work as before with identical outputs and `state_dict()` compatibility.
-
----
-
-## [1.0.8.post1] - 2026-10-30
-
-### Fixed
-- **PyPI Discovery Description & Package Metadata**:
-  * Updated `pyproject.toml` distribution `description` to prominently highlight LuminaV low-precision BF16/FP16 optimization, zero-copy multimodal dataset streaming, and native Rust SIMD, Vulkan, and Triton acceleration.
-- **Un-fused Bucket Cap Configuration Warning**:
-  * Added protective `UserWarning` in `LuminaV.__init__` when `bucket_cap_mb` is explicitly configured while `fuse_tensors=False`, preventing developer confusion regarding memory bounding scope on uncoalesced parameter tensors.
-
----
-
-## [1.0.8] - 2026-10-29
-
-### Added
-- **LuminaV v1.5.5 Contiguous Bucket Fusion (`fuse_tensors=False`) & Triton JIT Pre-Warmup (`prewarm_jit=True`)**:
-  * Added opt-in `fuse_tensors: bool = False` in `LuminaV` to coalesce fragmented parameter groups into contiguous 1D buffers (`flat_p`, `flat_g`, `flat_m`, `flat_v`, `flat_mp`) with zero-copy tensor views, in-place `zero_grad()` pointer preservation, and LSO-Eval Stage II compliant `state_dict()` / `load_state_dict()` round-tripping.
-  * Added `prewarm_jit: bool = True` (enabled by default) in `LuminaVGpuMixin._prewarm_all_configs` to pre-compile Triton GPU kernels (`step=1` two-pass and `step=2+` fused single-pass + post-step EMA) on 64-element dummy tensors (`lr=0.0`) during `__init__()`.
-  * Isolated `_prepare_triton_buffers` state lookup via `self.state.get(p)` to prevent contiguous bucket or warmup buffers from polluting `self.state`.
-- **Bounded Bucket Chunking (`bucket_cap_mb = 500.0`) & Sequential In-Place Migration**:
-  * Added `bucket_cap_mb: float = 500.0` in `LuminaV` to partition contiguous parameter coalescing into bounded chunk buckets (maximum 500 MB per bucket), preventing monolithic VRAM allocation spikes on large models (e.g. 4B+).
-  * Implemented `_partition_into_bounded_chunks` with isolated single-tensor handling for huge layers (e.g. large embedding matrices).
-  * Implemented sequential in-place migration per parameter inside chunks with immediate dereferencing (`del old_data`, `del old_grad`, `del conv_m`, `del conv_v`) and `gc.collect()`, ensuring that peak transient memory spikes during initialization never exceed 1 chunk (max 500 MB).
-
----
-
-## [1.0.7] - 2026-10-29
-
-### Added
-- **LuminaV v1.5.4 Specification Upgrade**: Upgraded the bundled LuminaV Optimizer specification to `v1.5.4`, removing `centered_defect` and its global reduction kernels (`_lumina_diff_sum_kernel`, `diff_sum.comp`, `reduce_diff_sum_simd`) across Rust core, SIMD, Vulkan 1.3/1.4, Triton AOT/JIT, PyTorch, and NumPy backends to reduce per-step latency.
-- **In-Container Native Bitwise-BF16 State Promotion**: Added native Bitwise-BF16 state promotion for pure `FP16` execution (`dtype_mode == 2`, `master_weights=False`) across Rust core, AVX-512/AVX2 SIMD, Vulkan 1.3/1.4 compute shaders, Triton kernels, PyTorch fallback, and standalone NumPy, preserving the 16-bit container memory footprint (zero extra VRAM) while expanding state dynamic range to 8-bit exponent ($1.18 \times 10^{-38}$).
-- **Cross-Precision Checkpoint Handshake**: Added `_convert_torch_state_buffer` in `luminamoon.optimizer.luminav` for bit-exact state conversions during `load_state_dict` and `step` between `master_weights=False` (`torch.float16` Bitwise-BF16) and `master_weights="full"` (`torch.float32` / `torch.bfloat16`).
-- **Comprehensive Hardware Parity & Regression Tests**: Added `test_vulkan_parity_dual_fp16_native_bf16_state_131072_5steps` and `test_vulkan_parity_tdclm_fp16_native_bf16_state_131072_5steps` in `vulkan_parity.rs` and `run_pure_fp16_native_bf16_state_tests` in `test_end_to_end_training.py`.
-
-### Fixed
-- **FP16 Convergence Stability & Loss Reset Mitigation**: Resolved sudden loss reset to $\ln(V)$ during convergence in pure `FP16` mode by eliminating second-moment underflow and unifying adaptive denominator epsilon (`effective_eps = eps = 1e-8`) across all execution backends (fixes #65).
-- **Scalar Reference Parity Synchronization**: Fixed state quantization mode propagation in CPU scalar reference `fused_single_pass` path (`dual.rs` and `tdclm.rs`), maintaining mathematical parity between CPU scalar and GPU Vulkan shaders.
-- **Headless CI Vulkan Detection Guard**: Added `LuminaV.detect_hardware()["vulkan_available"]` check in end-to-end tests to prevent false failures on headless CPU runners.
-
----
-
-## [1.0.6.post1] - 2026-10-26
-
-### Fixed
-- **Prefetcher Batch ID Synchronization**: Fixed batch ID drift in `ResumablePipeline` during async prefetching (`prefetch_batches > 0`) by tagging each queue item with `(batch_payload, batch_id)` and synchronizing `self._active_batch_id` on consumption, eliminating premature coordinate commits and `UnknownBatchId` errors.
-- **Data Engine Fallback Parity**: Fixed `StreamDeduplicator.count` callable signature in `luminamoon.data.fallbacks.dedup` (and high-level wrapper polymorphic dispatch), restored RAM staging in `StateTracker.update(..., batch_id=1)`, and aligned `fused_pipeline_process_batch` with two-stage staging and `commit_dedup` gating.
-- **Optimizer Hyperparameter Constants**: Explicitly populated complete default hyperparameter constants in `luminamoon.optimizer.constants`.
-- **Version Synchronization Tooling**: Added `--check` mode and PEP 440 validation in `tools/sync_version.py` to prevent accidental version string corruption.
-
----
-
-## [1.0.6] - 2026-10-26
-
-### Added
-- **Python 3.15 Classifier Support**: Added `Programming Language :: Python :: 3.15` to package metadata in `pyproject.toml`.
-- **O(1) Deduplication Membership Check**: Added `TwoStageDeduplicator::is_seen` in `data/src/dedup.rs` and wired `PyStreamDeduplicator::is_seen` to avoid $O(N)$ hash-vector exports.
-
-### Changed
-- **PROJECT LOOMY Rebranding & Author Standardization**: Standardized project branding to **PROJECT LOOMY** (`Loomy(luminamoon)`) with **Silver Moon (`@cloverxion`)** listed first as lead author followed by **Lumina Moon (`@cloverx-id`) contributors** across documentation and manifests.
-- **Optional Dependency Import Diagnostics**: Wrapped `numpy`, `torch`, `pyarrow`, and `polars` imports in `StringTensor` export methods with actionable installation hints.
-- **Fallback Signature Parity**: Aligned `suggest_closest_matches` (with Damerau-Levenshtein edit-distance ranking) and `postprocess_and_dedup_records` return tuple `(cleaned_unique, newly_committed)` in `luminamoon.data.fallbacks` with Rust FFI and `_stubs.py`.
-
-### Fixed
-- **WAL & GPU Buffer Corruption Hardening**: Added byte-alignment validation (`% 8 == 0` for `.dedup` WAL files, `% 4 == 0` for GPU `uint32` offset buffers), exact pre-commit offset seeking in `DualMetadataManager::commit_gpu_batch`, and `StringView16` field encapsulation (`pub(crate)`), inline length clamping, and saturating `u32` offset arithmetic in `data/src/string_tensor.rs`.
-- **Data Pipeline & Collate Hardening**: Fixed WAL double-commit skip in `fused_process_batch`, added missing column typo suggestions and `bytes` decoding in `_extract_text`, auto-populated `tokenizer.pad_token = tokenizer.eos_token` when missing, restored resume coordinates in `ResumablePipeline`, and eliminated background prefetcher queue timeout drops.
-
----
-
-## [1.0.5.post1] - 2026-10-25
-
-### Changed
-- **Modular Data Processing Engine**: Decomposed the monolithic data processing fallback layer in `luminamoon.data` into dedicated submodules under `fallbacks/` (`cleaner.py`, `quality.py`, `dedup.py`, `minhash.py`, `masker.py`, `coordinate.py`, `tracker.py`, `string_tensor.py`, `fused.py`, `serialization.py`, `helpers.py`), preserving backward compatibility.
-- **Updated Algorithm Changelog**: Updated `LUMINAV_ALGORITHM_CHANGELOG.md` with detailed mathematical formulation and parameter comparisons between 1.3.3 and 1.5.3.
-- **PEP 440 Post-Release Synchronization**: Bumped package distribution version to `1.0.5.post1`.
-
----
-
-## [1.0.5] - 2026-10-24
-
-### Added
-- **Comprehensive In-Editor Docstrings (`luminav.py`)**: Added Google-style docstrings across `LuminaV.__init__`, `step`, `zero_grad`, `state_dict`, `load_state_dict`, and `detect_hardware`, providing full IDE autocomplete tooltips and explicit default hyperparameter values.
-- **Hyperparameter Quick Reference (`README.md`)**: Added a structured default parameter reference table directly to the main repository README.
-- **Dedicated Dual-Track Changelog Directory (`changelogs/`)**: Separated release documentation into `SOFTWARE_CHANGELOG.md` (package releases) and `LUMINAV_ALGORITHM_CHANGELOG.md` (algorithm specification history).
-- **Explicit Algorithm Specification Constant (`LuminaV.ALGORITHM_SPEC`)**: Added `LuminaV.ALGORITHM_SPEC = "1.5.3"` to clearly distinguish the mathematical specification version from the software package version.
-
-### Changed
-- **Unified Software Version Synchronization (`1.0.5`)**: Synchronized `luminamoon.__version__`, `luminamoon.data.get_version()`, `pyproject.toml`, and `Cargo.toml` to a single source of truth (`1.0.5`).
-- **Offline Version Resolution (`version.py`)**: Removed runtime HTTP GitHub file fetching from `luminamoon.optimizer.version` and `luminamoon.data.version`, allowing offline execution in air-gapped HPC clusters.
-
-### Security
-- **Upgraded `setuptools` (`84.0.0` -> `84.1.0`)**: Patched development dependency `setuptools` in `requirements.txt` to resolve `GHSA-xmj3-9gf7-h52w` (`CWE-22`).
-
----
-
-## [1.0.4] - 2026-10-24
-
-### Added
-- **Free-Threaded CPython (No-GIL / PEP 703) Support**: Verified compilation and runtime execution of native `abi3t` / free-threaded wheels on Python `3.14t` and `3.15t`.
-- **Rust Core Invariant Tests**: Added `optimizer/crates/luminav-core/tests/core_tests.rs` covering PRNG seed determinism, Padé rational $\tanh$ bounds, scratch buffer EMA smoothing, and Levenshtein typo suggestions.
-- **PyPI Project URLs & Contributors Metadata**: Added `Homepage`, `Repository`, `Issues`, `Documentation`, and `Changelog` links alongside `Lumina Moon (cloverx-id) Contributors` to `pyproject.toml`, `Cargo.toml`, and `LICENSE`.
-
-### Changed
-- **Rust Workspace Modularization**:
-  - Decomposed `luminav-core/src/reference_cpu.rs` into `reference_cpu/{mod, dual, tdclm}.rs`.
-  - Decomposed `luminav-simd` (`avx2.rs`, `avx512.rs`, `parallel.rs`) into dedicated submodule directories (`dual.rs`, `tdclm.rs`, `mod.rs`).
-  - Extracted Vulkan UMA step dispatches into `luminav-vulkan/src/{dual, tdclm}.rs`.
-  - Decomposed `optimizer/src/python_ffi.rs` into `python_ffi/{config_parser, extract, worker, mod}.rs`.
-- **Python Package Modularization**:
-  - Split `luminav.py` into `standalone.py`, `validation.py`, and `numpy_backend.py`.
-  - Split `gpu_backend.py` into `fsdp_sync.py`, `torch_ops.py`, and `triton_kernels.py`.
-  - Added cross-platform cache directory resolution (`LOCALAPPDATA` on Windows, `XDG_CACHE_HOME` on POSIX).
-- **Reference Implementation Restructuring**:
-  - Decomposed monolithic `optimizer/reference/luminav_v153.py` (3,996 lines) into `core/`, `kernels/`, `single_backend/`, `triton_backend/`, and `utils/`.
-
----
-
-## [1.0.3] - 2026-10-21
-
-### Added
-- **Two-Stage Deferred Release Pipeline**: Implemented draft GitHub release artifact staging on tag push (`v*`) with automated PyPI publication triggered upon release publication.
-
-### Fixed
-- **SIMD Finite BF16 Clamps**: Hardened AVX-512 and AVX2 finite clamp boundaries prior to BFloat16 stochastic rounding.
-- **Build Wheel Privacy Sanitizer**: Added automated local path scrubbing in `tools/build_wheel.py` without modifying compiled ELF/PE/Mach-O relocation tables.
-
----
-
-## [1.0.2.post2] - 2026-10-20
-
-### Fixed
-- **Dynamic Tag Versioning**: Added `tools/sync_version.py` to synchronize `pyproject.toml` and `_version.py` directly from Git release tags during CI wheel builds.
-
----
-
-## [1.0.2.post1] - 2026-10-20
-
-### Fixed
-- **PEP 440 Post-Release Compliance**: Standardized version regex and enabled `skip-existing: true` on PyPI publishing workflows.
-
----
-
-## [1.0.2] - 2026-10-20
-
-### Added
-- **Modular `luminamoon.data` Engine**: Added SIMD text normalization (`TextCleaner`), Shannon entropy & symbol heuristic filtering (`QualityFilter`), two-stage XXH3-64 exact deduplication (`StreamDeduplicator`), 64-permutation MinHash LSH fuzzy deduplication (`MinHashDeduplicator`), PII masking (`PIIMasker`), 128-byte Ping-Pong crash recovery (`StateTracker`), and dual-layout `StringTensor` (CPU 16-byte Umbra `StringView` & GPU contiguous offset+payload buffers).
-- **LuminaV v1.5.3 Engine Integration**: Integrated the Rust AVX-512/AVX2 SIMD, Vulkan 1.3/1.4 UMA compute, and Triton AOT/JIT backends.
-
----
-
-## [1.0.1] - 2026-10-20
-
-### Added
-- Initial monorepo release unifying the `luminamoon` Universal DataLoader and `LuminaV` Optimizer under a single Rust + Python workspace.
+# LuminaMoon Software Changelog
+
+All notable software, packaging, FFI, and infrastructure changes to the `luminamoon` monorepo and PyPI distribution (`pip install luminamoon`) are documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+For the mathematical and kernel evolution of the **LuminaV Optimizer** specification (`v1.0.0` to `v1.6.0`), see [`LUMINAV_ALGORITHM_CHANGELOG.md`](LUMINAV_ALGORITHM_CHANGELOG.md).
+
+---
+
+## [1.1.0.post3] - 2026-11-07
+
+### Changed
+- **Academic Citation Consolidation**:
+  * Removed redundant `CITATION.cff` metadata file to consolidate citation records directly into Zenodo (DOI: [`10.5281/zenodo.23978801`](https://doi.org/10.5281/zenodo.23978801)) and repository `README.md`.
+- **Data Engine Version Introspection (`python/luminamoon/data/version.py`)**:
+  * Streamlined data version checking so `get_version()` directly returns the active package version string without disk or mutable state side effects.
+  * Maintained full backward compatibility for `list_versions()` and `set_version()` with informative `DeprecationWarning` notices explaining that the data engine operates on the unified native SIMD implementation.
+  * Removed internal cache directory cloning and dynamic module loading routines from `~/.cache/luminamoon/versions`.
+  * Updated `data/README.md` to document direct version inspection via `luminamoon.data.get_version()`.
+- **Documentation & Technical Guides**:
+  * Rewrote technical guides, component documentation, and reference overviews across `docs/` and `data/docs/` in direct, humble, and accessible language for human readability.
+  * Preserved core algorithmic identities including the Triangle Family, Triangle Inequality Envelope, Gradient-Momentum Residual, and Triangle-Defect Penalty without inflated descriptors.
+  * Normalized data engine documentation in `python/luminamoon/data` and `data/` to consistent English terminology.
+  * Aligned the MinHash documentation in `docs/dataloader/sources_and_streaming/README.md` with the Rust implementation using XXH3-64 seed multipliers.
+
+### Refined
+- **Evaluation Specification Artifacts (`eval/`)**:
+  * Updated `eval/lso_eval_specification.tex` with refined layout and mathematical notation.
+  * Added updated compiled specification manuscript `eval/LSO_Eval.pdf` and removed legacy PDF files.
+  * Aligned `tools/sync_huggingface.py` and `eval/README.md` to track `eval/LSO_Eval.pdf`.
+- **PosRaGLU Activation Gating (`luminamoon.nn`)**:
+  * Added explicit positive anchor support ($b > 0$) for `pos_ra_gate` and `PosRaGLU`, preserving unit derivative at the origin while bounding output in $(0, 1)$.
+
+### Fixed
+- **Markdown Parameter Table Pipe Escaping**:
+  * Escaped pipe characters in inline code notation (`\|d_t\| = \|g_t - m_{t-1}\|`) within the README parameter table to prevent GitHub Flavored Markdown (GFM) table parsers from prematurely splitting table cells.
+- **Static Type Analysis (`luminamoon.nn._compat`)**:
+  * Declared `_rs_engine` at module scope with multi-tier import fallbacks to eliminate Pyright unresolved import warnings.
+
+### Infrastructure
+- **Continuous Integration & Release Automation**:
+  * Added Python runtime setup to the Rust test workflow in `.github/workflows/ci.yml` so linkers reliably locate shared Python libraries during PyO3 FFI tests on updated runner images.
+  * Streamlined GitHub Release note generation by eliminating redundant boilerplate headers and prioritizing direct changelog entries.
+
+---
+
+## [1.1.0.post2] - 2026-11-05
+
+### Fixed
+- **Rust Arrow & Parquet Dependencies (`RUSTSEC-2026-0442`)**:
+  * Updated `parquet`, `arrow-array`, `arrow-schema`, and `arrow-ipc` to `60.1.0` in workspace manifests (`Cargo.toml` and `Cargo.lock`).
+  * Resolves advisory **RUSTSEC-2026-0442** (CVE-2026-111959) in `arrow-data`, preventing potential out-of-bounds reads when validating RunEndEncoded arrays during dataset ingestion.
+
+---
+
+## [1.1.0] - 2026-11-04
+
+### Added
+- **Hardware-Accelerated Triangle Family Fused Triton Kernels (GPU JIT & Rust AOT)**:
+  * **Python JIT (`_lumina_t1_single_pass_kernel`, `_lumina_t2_fused_single_pass_kernel`)**: Added dedicated fused register kernels for the Triangle Family (`buffer="T1"` and `buffer="T2"`), replacing eager PyTorch chunking and eliminating CPU-GPU `.item()` sync stalls, bringing T2 execution latency from 20.7 ms down to native GPU speed (~0.9–1.0 ms).
+  * **Rust Triton AOT Crate (`luminav-triton-aot`)**: Added Ahead-Of-Time compiled kernels for `T1` and `T2` across 12 architectures (Turing, Ampere, Ada, Hopper, Blackwell, CDNA, RDNA3), expanding the embedded manifest to 96 binary artifacts.
+- **Strict Version Capability Enforcement**:
+  * Enforced specification capabilities across historical LuminaV algorithm versions (`1.6.0`, `1.5.5`, `1.5.4`, `1.5.3`): locking to `1.5.4` or `1.5.5` strictly rejects features introduced in later versions (such as `buffer="T2"` or `fuse_tensors=True`), guaranteeing reproducibility for academic benchmarks.
+
+### Added
+- **Ra-Family (Rational, Tanh-Free & Square-Root-Free Neural Architecture)**:
+  * **Zero Mandatory PyTorch Dependency**: The `luminamoon` package remains lightweight (~2.4 MB) with zero required dependencies (`dependencies = []`). The new `luminamoon.nn` module automatically detects the runtime environment (`_compat.py`), inheriting from `torch.nn.Module` when PyTorch is installed, or falling back cleanly to standalone NumPy buffers when PyTorch is absent.
+  * **`RaNorm` (`luminamoon.nn.RaNorm`)**: Token normalization using $L_1$ mean absolute deviation instead of root-mean-square. Eliminates square-root evaluations and amortizes a single scalar reciprocal per token vector.
+  * **`RaBatchNorm2d` (`luminamoon.nn.RaBatchNorm2d`)**: 2D batch normalization using cross-batch mean absolute deviation (MAD). Includes inference weight folding (`fold_into_conv`) that merges running statistics into preceding `Conv2d` layers with zero additional inference latency.
+  * **`RaTanh` & `ra_tanh`**: Rational hyperbolic tangent with non-vanishing algebraic gradient tails $O(1 / |x|^3)$ and unit derivative at origin.
+  * **`RaGELU` & `ra_gelu`**: Rational GELU activation using rational polynomial approximations.
+  * **`PosRaGLU` (`luminamoon.nn.PosRaGLU`)**: Gated Linear Unit with strictly positive rational gating bounded in $(0, 1)$. Eliminates quadratic activation outlier explosion for stable low-precision (INT4/FP8) quantization.
+  * **`RaGLU` & `TriGLU`**: Standard and triple rational gated linear unit variants.
+  * **`RaAttention` & `RaAttn` (`luminamoon.nn.RaAttention`, `luminamoon.nn.RaAttn`)**: Multi-head attention using Cayley-Padé rational softmax. Supports causal masking, QK-Norm (`qk_norm=True`), dropout, and custom head dimension scaling.
+  * **`ra_softmax` (`luminamoon.nn.ra_softmax`)**: Rational softmax with exact compact support for $\Delta \ge 16$, guaranteeing sum of numerators $\ge 1.0$ without requiring an epsilon floor.
+  * **Dual Hardware Acceleration (Triton JIT GPU & Rust AVX-512/AVX2 SIMD CPU)**:
+    - Added Triton GPU kernels (`_ra_norm_fwd_kernel`, `_pos_ra_glu_fwd_kernel`, `_ra_softmax_fwd_kernel`) in `luminamoon.nn.triton_kernels` for register-fused execution without intermediate global memory roundtrips.
+    - Added native Rust SIMD kernels in `luminav-simd` with AVX2/FMA vectorization and PyO3 bindings for high-throughput CPU token normalization and rational softmax.
+- **Academic Aliases (`luminamoon.nn`)**:
+  * Exported standardized academic aliases across neural components: `L1Norm`, `L1LayerNorm`, `L1BatchNorm2d`, `RationalTanh`, `rational_tanh`, `RationalGELU`, `rational_gelu`, `RationalGLU`, `BoundedRationalGLU`, `rational_softmax`, `pade_softmax`, `RationalAttention`, and `PadeAttention`.
+- **Theoretical References & Literature Documentation**:
+  * Added `references/README.md` and `references/bibliography.bib` documenting literature mappings and rational approximations, paired with `docs/references.md` in the documentation.
+- **Empirical Evaluation**:
+  * Updated `eval/LSO-Eval.pdf` with the latest benchmark convergence findings and empirical evaluation report.
+- **Unit Tests**: Added 32 unit tests in `tests/test_nn_ra_family.py` covering mathematical invariants, convolution folding, isolated zero-torch subprocess execution, and hardware introspection.
+
+### Changed
+- **Security Policy**: Streamlined `SECURITY.md` supported version matrix (`luminamoon >= 1.1.0`, `luminav >= 1.6.0`) with simplified reporting guidelines.
+
+---
+
+## [1.0.9] - 2026-11-03
+
+### Added
+- **LuminaV v1.6.0 — T-FAMILY (Triangle Family) tanh-free Modes (`buffer="T1"` & `buffer="T2"`)**:
+  * **`buffer="T2"` (2-Buffer Triangle Family Mode)**: Keeps the 2-buffer state (`exp_avg` and `exp_avg_sq`) without using $\tanh$. Computes the gradient-momentum residual $d_t = g_t - m _ {t-1}$ and bounds the lookahead update via the **Triangle Inequality Envelope**:
+
+    $\displaystyle \mathcal{E} _ t^{(\text{T2})} = w_1 |m _ {t-1}| + w_2 |g_t| + 0.5\lambda |d_t| + 0.5\lambda \sigma_t + \epsilon _ {\text{eff}}, \qquad u_t^{(\text{T2})} = \frac{\tilde{m} _ t}{\mathcal{E} _ t^{(\text{T2})}} \in (-1, 1)$
+
+    By the Triangle Inequality, $|w_1 m _ {t-1} + w_2 g_t| \le w_1 |m _ {t-1}| + w_2 |g_t| \le \mathcal{E} _ t^{(\text{T2})}$, which guarantees the update step $u_t^{(\text{T2})}$ strictly remains within $(-1, 1)$ without evaluating $\tanh$. The term $0.5\lambda |d_t|$ acts as a **triangle-defect penalty**, while $0.5\lambda \sigma_t$ dampens belief-variance dispersion.
+  * **`buffer="T1"` (1-Buffer Triangle Family Mode)**: Pairs the single-buffer **Triangle-Defect** denominator ($\mathcal{E} _ t^{(\text{T1})} = w_1 |m _ {t-1}| + w_2 |g_t| + \lambda |d_t| + \epsilon _ {\text{eff}}$ with residual $d_t = g_t - m _ {t-1}$) with the rational update bounds below via the Triangle Inequality, eliminating the second-moment accumulator ($v_t$) and reducing optimizer state memory by 50%.
+  * **Rational Update Bounds (for `T1` & `T2`)**: Replaces $\tanh$ and $\sqrt{\cdot}$ in the coordinate, radial, and FP16 update bounds with simple rational formulas across Rust, PyTorch, FSDP, and NumPy:
+
+    $\displaystyle b_i = \mathrm{bound} _ {\text{ratio}} \cdot (|w_i| + 10^{-2}), \qquad \Delta w_i = \frac{b_i (b_i + |s_i|) s_i}{b_i (b_i + |s_i|) + s_i^2}, \qquad \mathrm{scale} _ {\text{radial}} = \frac{1 + r}{1 + r + r^2}$
+- **100% Backward Compatibility & Zero Breaking Changes**:
+  * Existing code and checkpoints using `buffer=2` (default), `buffer=1`, or `version="1.5.5"` continue to work as before with identical outputs and `state_dict()` compatibility.
+
+---
+
+## [1.0.8.post1] - 2026-10-30
+
+### Fixed
+- **PyPI Discovery Description & Package Metadata**:
+  * Updated `pyproject.toml` distribution `description` to prominently highlight LuminaV low-precision BF16/FP16 optimization, zero-copy multimodal dataset streaming, and native Rust SIMD, Vulkan, and Triton acceleration.
+- **Un-fused Bucket Cap Configuration Warning**:
+  * Added protective `UserWarning` in `LuminaV.__init__` when `bucket_cap_mb` is explicitly configured while `fuse_tensors=False`, preventing developer confusion regarding memory bounding scope on uncoalesced parameter tensors.
+
+---
+
+## [1.0.8] - 2026-10-29
+
+### Added
+- **LuminaV v1.5.5 Contiguous Bucket Fusion (`fuse_tensors=False`) & Triton JIT Pre-Warmup (`prewarm_jit=True`)**:
+  * Added opt-in `fuse_tensors: bool = False` in `LuminaV` to coalesce fragmented parameter groups into contiguous 1D buffers (`flat_p`, `flat_g`, `flat_m`, `flat_v`, `flat_mp`) with zero-copy tensor views, in-place `zero_grad()` pointer preservation, and LSO-Eval Stage II compliant `state_dict()` / `load_state_dict()` round-tripping.
+  * Added `prewarm_jit: bool = True` (enabled by default) in `LuminaVGpuMixin._prewarm_all_configs` to pre-compile Triton GPU kernels (`step=1` two-pass and `step=2+` fused single-pass + post-step EMA) on 64-element dummy tensors (`lr=0.0`) during `__init__()`.
+  * Isolated `_prepare_triton_buffers` state lookup via `self.state.get(p)` to prevent contiguous bucket or warmup buffers from polluting `self.state`.
+- **Bounded Bucket Chunking (`bucket_cap_mb = 500.0`) & Sequential In-Place Migration**:
+  * Added `bucket_cap_mb: float = 500.0` in `LuminaV` to partition contiguous parameter coalescing into bounded chunk buckets (maximum 500 MB per bucket), preventing monolithic VRAM allocation spikes on large models (e.g. 4B+).
+  * Implemented `_partition_into_bounded_chunks` with isolated single-tensor handling for huge layers (e.g. large embedding matrices).
+  * Implemented sequential in-place migration per parameter inside chunks with immediate dereferencing (`del old_data`, `del old_grad`, `del conv_m`, `del conv_v`) and `gc.collect()`, ensuring that peak transient memory spikes during initialization never exceed 1 chunk (max 500 MB).
+
+---
+
+## [1.0.7] - 2026-10-29
+
+### Added
+- **LuminaV v1.5.4 Specification Upgrade**: Upgraded the bundled LuminaV Optimizer specification to `v1.5.4`, removing `centered_defect` and its global reduction kernels (`_lumina_diff_sum_kernel`, `diff_sum.comp`, `reduce_diff_sum_simd`) across Rust core, SIMD, Vulkan 1.3/1.4, Triton AOT/JIT, PyTorch, and NumPy backends to reduce per-step latency.
+- **In-Container Native Bitwise-BF16 State Promotion**: Added native Bitwise-BF16 state promotion for pure `FP16` execution (`dtype_mode == 2`, `master_weights=False`) across Rust core, AVX-512/AVX2 SIMD, Vulkan 1.3/1.4 compute shaders, Triton kernels, PyTorch fallback, and standalone NumPy, preserving the 16-bit container memory footprint (zero extra VRAM) while expanding state dynamic range to 8-bit exponent ($1.18 \times 10^{-38}$).
+- **Cross-Precision Checkpoint Handshake**: Added `_convert_torch_state_buffer` in `luminamoon.optimizer.luminav` for bit-exact state conversions during `load_state_dict` and `step` between `master_weights=False` (`torch.float16` Bitwise-BF16) and `master_weights="full"` (`torch.float32` / `torch.bfloat16`).
+- **Comprehensive Hardware Parity & Regression Tests**: Added `test_vulkan_parity_dual_fp16_native_bf16_state_131072_5steps` and `test_vulkan_parity_tdclm_fp16_native_bf16_state_131072_5steps` in `vulkan_parity.rs` and `run_pure_fp16_native_bf16_state_tests` in `test_end_to_end_training.py`.
+
+### Fixed
+- **FP16 Convergence Stability & Loss Reset Mitigation**: Resolved sudden loss reset to $\ln(V)$ during convergence in pure `FP16` mode by eliminating second-moment underflow and unifying adaptive denominator epsilon (`effective_eps = eps = 1e-8`) across all execution backends (fixes #65).
+- **Scalar Reference Parity Synchronization**: Fixed state quantization mode propagation in CPU scalar reference `fused_single_pass` path (`dual.rs` and `tdclm.rs`), maintaining mathematical parity between CPU scalar and GPU Vulkan shaders.
+- **Headless CI Vulkan Detection Guard**: Added `LuminaV.detect_hardware()["vulkan_available"]` check in end-to-end tests to prevent false failures on headless CPU runners.
+
+---
+
+## [1.0.6.post1] - 2026-10-26
+
+### Fixed
+- **Prefetcher Batch ID Synchronization**: Fixed batch ID drift in `ResumablePipeline` during async prefetching (`prefetch_batches > 0`) by tagging each queue item with `(batch_payload, batch_id)` and synchronizing `self._active_batch_id` on consumption, eliminating premature coordinate commits and `UnknownBatchId` errors.
+- **Data Engine Fallback Parity**: Fixed `StreamDeduplicator.count` callable signature in `luminamoon.data.fallbacks.dedup` (and high-level wrapper polymorphic dispatch), restored RAM staging in `StateTracker.update(..., batch_id=1)`, and aligned `fused_pipeline_process_batch` with two-stage staging and `commit_dedup` gating.
+- **Optimizer Hyperparameter Constants**: Explicitly populated complete default hyperparameter constants in `luminamoon.optimizer.constants`.
+- **Version Synchronization Tooling**: Added `--check` mode and PEP 440 validation in `tools/sync_version.py` to prevent accidental version string corruption.
+
+---
+
+## [1.0.6] - 2026-10-26
+
+### Added
+- **Python 3.15 Classifier Support**: Added `Programming Language :: Python :: 3.15` to package metadata in `pyproject.toml`.
+- **O(1) Deduplication Membership Check**: Added `TwoStageDeduplicator::is_seen` in `data/src/dedup.rs` and wired `PyStreamDeduplicator::is_seen` to avoid $O(N)$ hash-vector exports.
+
+### Changed
+- **PROJECT LOOMY Rebranding & Author Standardization**: Standardized project branding to **PROJECT LOOMY** (`Loomy(luminamoon)`) with **Silver Moon (`@cloverxion`)** listed first as lead author followed by **Lumina Moon (`@cloverx-id`) contributors** across documentation and manifests.
+- **Optional Dependency Import Diagnostics**: Wrapped `numpy`, `torch`, `pyarrow`, and `polars` imports in `StringTensor` export methods with actionable installation hints.
+- **Fallback Signature Parity**: Aligned `suggest_closest_matches` (with Damerau-Levenshtein edit-distance ranking) and `postprocess_and_dedup_records` return tuple `(cleaned_unique, newly_committed)` in `luminamoon.data.fallbacks` with Rust FFI and `_stubs.py`.
+
+### Fixed
+- **WAL & GPU Buffer Corruption Hardening**: Added byte-alignment validation (`% 8 == 0` for `.dedup` WAL files, `% 4 == 0` for GPU `uint32` offset buffers), exact pre-commit offset seeking in `DualMetadataManager::commit_gpu_batch`, and `StringView16` field encapsulation (`pub(crate)`), inline length clamping, and saturating `u32` offset arithmetic in `data/src/string_tensor.rs`.
+- **Data Pipeline & Collate Hardening**: Fixed WAL double-commit skip in `fused_process_batch`, added missing column typo suggestions and `bytes` decoding in `_extract_text`, auto-populated `tokenizer.pad_token = tokenizer.eos_token` when missing, restored resume coordinates in `ResumablePipeline`, and eliminated background prefetcher queue timeout drops.
+
+---
+
+## [1.0.5.post1] - 2026-10-25
+
+### Changed
+- **Modular Data Processing Engine**: Decomposed the monolithic data processing fallback layer in `luminamoon.data` into dedicated submodules under `fallbacks/` (`cleaner.py`, `quality.py`, `dedup.py`, `minhash.py`, `masker.py`, `coordinate.py`, `tracker.py`, `string_tensor.py`, `fused.py`, `serialization.py`, `helpers.py`), preserving backward compatibility.
+- **Updated Algorithm Changelog**: Updated `LUMINAV_ALGORITHM_CHANGELOG.md` with detailed mathematical formulation and parameter comparisons between 1.3.3 and 1.5.3.
+- **PEP 440 Post-Release Synchronization**: Bumped package distribution version to `1.0.5.post1`.
+
+---
+
+## [1.0.5] - 2026-10-24
+
+### Added
+- **Comprehensive In-Editor Docstrings (`luminav.py`)**: Added Google-style docstrings across `LuminaV.__init__`, `step`, `zero_grad`, `state_dict`, `load_state_dict`, and `detect_hardware`, providing full IDE autocomplete tooltips and explicit default hyperparameter values.
+- **Hyperparameter Quick Reference (`README.md`)**: Added a structured default parameter reference table directly to the main repository README.
+- **Dedicated Dual-Track Changelog Directory (`changelogs/`)**: Separated release documentation into `SOFTWARE_CHANGELOG.md` (package releases) and `LUMINAV_ALGORITHM_CHANGELOG.md` (algorithm specification history).
+- **Explicit Algorithm Specification Constant (`LuminaV.ALGORITHM_SPEC`)**: Added `LuminaV.ALGORITHM_SPEC = "1.5.3"` to clearly distinguish the mathematical specification version from the software package version.
+
+### Changed
+- **Unified Software Version Synchronization (`1.0.5`)**: Synchronized `luminamoon.__version__`, `luminamoon.data.get_version()`, `pyproject.toml`, and `Cargo.toml` to a single source of truth (`1.0.5`).
+- **Offline Version Resolution (`version.py`)**: Removed runtime HTTP GitHub file fetching from `luminamoon.optimizer.version` and `luminamoon.data.version`, allowing offline execution in air-gapped HPC clusters.
+
+### Security
+- **Upgraded `setuptools` (`84.0.0` -> `84.1.0`)**: Patched development dependency `setuptools` in `requirements.txt` to resolve `GHSA-xmj3-9gf7-h52w` (`CWE-22`).
+
+---
+
+## [1.0.4] - 2026-10-24
+
+### Added
+- **Free-Threaded CPython (No-GIL / PEP 703) Support**: Verified compilation and runtime execution of native `abi3t` / free-threaded wheels on Python `3.14t` and `3.15t`.
+- **Rust Core Invariant Tests**: Added `optimizer/crates/luminav-core/tests/core_tests.rs` covering PRNG seed determinism, Padé rational $\tanh$ bounds, scratch buffer EMA smoothing, and Levenshtein typo suggestions.
+- **PyPI Project URLs & Contributors Metadata**: Added `Homepage`, `Repository`, `Issues`, `Documentation`, and `Changelog` links alongside `Lumina Moon (cloverx-id) Contributors` to `pyproject.toml`, `Cargo.toml`, and `LICENSE`.
+
+### Changed
+- **Rust Workspace Modularization**:
+  - Decomposed `luminav-core/src/reference_cpu.rs` into `reference_cpu/{mod, dual, tdclm}.rs`.
+  - Decomposed `luminav-simd` (`avx2.rs`, `avx512.rs`, `parallel.rs`) into dedicated submodule directories (`dual.rs`, `tdclm.rs`, `mod.rs`).
+  - Extracted Vulkan UMA step dispatches into `luminav-vulkan/src/{dual, tdclm}.rs`.
+  - Decomposed `optimizer/src/python_ffi.rs` into `python_ffi/{config_parser, extract, worker, mod}.rs`.
+- **Python Package Modularization**:
+  - Split `luminav.py` into `standalone.py`, `validation.py`, and `numpy_backend.py`.
+  - Split `gpu_backend.py` into `fsdp_sync.py`, `torch_ops.py`, and `triton_kernels.py`.
+  - Added cross-platform cache directory resolution (`LOCALAPPDATA` on Windows, `XDG_CACHE_HOME` on POSIX).
+- **Reference Implementation Restructuring**:
+  - Decomposed monolithic `optimizer/reference/luminav_v153.py` (3,996 lines) into `core/`, `kernels/`, `single_backend/`, `triton_backend/`, and `utils/`.
+
+---
+
+## [1.0.3] - 2026-10-21
+
+### Added
+- **Two-Stage Deferred Release Pipeline**: Implemented draft GitHub release artifact staging on tag push (`v*`) with automated PyPI publication triggered upon release publication.
+
+### Fixed
+- **SIMD Finite BF16 Clamps**: Hardened AVX-512 and AVX2 finite clamp boundaries prior to BFloat16 stochastic rounding.
+- **Build Wheel Privacy Sanitizer**: Added automated local path scrubbing in `tools/build_wheel.py` without modifying compiled ELF/PE/Mach-O relocation tables.
+
+---
+
+## [1.0.2.post2] - 2026-10-20
+
+### Fixed
+- **Dynamic Tag Versioning**: Added `tools/sync_version.py` to synchronize `pyproject.toml` and `_version.py` directly from Git release tags during CI wheel builds.
+
+---
+
+## [1.0.2.post1] - 2026-10-20
+
+### Fixed
+- **PEP 440 Post-Release Compliance**: Standardized version regex and enabled `skip-existing: true` on PyPI publishing workflows.
+
+---
+
+## [1.0.2] - 2026-10-20
+
+### Added
+- **Modular `luminamoon.data` Engine**: Added SIMD text normalization (`TextCleaner`), Shannon entropy & symbol heuristic filtering (`QualityFilter`), two-stage XXH3-64 exact deduplication (`StreamDeduplicator`), 64-permutation MinHash LSH fuzzy deduplication (`MinHashDeduplicator`), PII masking (`PIIMasker`), 128-byte Ping-Pong crash recovery (`StateTracker`), and dual-layout `StringTensor` (CPU 16-byte Umbra `StringView` & GPU contiguous offset+payload buffers).
+- **LuminaV v1.5.3 Engine Integration**: Integrated the Rust AVX-512/AVX2 SIMD, Vulkan 1.3/1.4 UMA compute, and Triton AOT/JIT backends.
+
+---
+
+## [1.0.1] - 2026-10-20
+
+### Added
+- Initial monorepo release unifying the `luminamoon` Universal DataLoader and `LuminaV` Optimizer under a single Rust + Python workspace.

config.json

@@ -1,122 +1,122 @@
-{
-  "name": "LuminaV",
-  "version": "1.6.0",
-  "package": "luminamoon",
-  "package_version": "1.0.9",
-  "description": "Master-free, low-precision BF16/FP16 deep learning optimizer with Dual-Buffer Convex Lookahead Momentum (CLM), Single-Buffer Triangle-Defect Convex Lookahead Momentum (TD-CLM) via the Triangle Inequality Envelope, rational Pade [13/6] bounding, stateless 32-bit MurmurHash3 stochastic rounding, and native Rust SIMD / Vulkan / Triton execution.",
-  "repository": "https://github.com/cloverx-id/luminamoon",
-  "documentation": "https://cloverx-id.github.io/luminamoon/",
-  "default_params": {
-    "lr": 0.0008,
-    "betas": [
-      0.9,
-      0.999
-    ],
-    "eps": 1e-08,
-    "weight_decay": 0.08,
-    "tau": 0.8,
-    "lam": 0.3,
-    "cautious": true,
-    "cautious_clamp_min": 0.5,
-    "buffer": 2,
-    "stochastic_rounding": true,
-    "bound": true,
-    "bound_type": "coordinate",
-    "bound_ratio": 0.05,
-    "master_weights": "none",
-    "execution": "auto",
-    "fused_single_pass": true,
-    "fuse_tensors": false,
-    "bucket_cap_mb": 500.0,
-    "prewarm_jit": true,
-    "ema_decay": "auto",
-    "fsdp_sync": false,
-    "seed": 1337
-  },
-  "presets": {
-    "dual_buffer_clm": {
-      "buffer": 2,
-      "tau": 0.8,
-      "cautious": true,
-      "cautious_clamp_min": 0.5,
-      "bound": true,
-      "bound_type": "coordinate",
-      "bound_ratio": 0.05,
-      "stochastic_rounding": true,
-      "master_weights": "none",
-      "fused_single_pass": true,
-      "ema_decay": "auto",
-      "fsdp_sync": false,
-      "description": "Default Dual-Buffer Convex Lookahead Momentum (CLM) tracking m_t and innovation variance v_t with coordinate trust-region bounding."
-    },
-    "single_buffer_td_clm": {
-      "buffer": 1,
-      "lam": 0.3,
-      "cautious": true,
-      "cautious_clamp_min": 0.5,
-      "bound": true,
-      "bound_type": "coordinate",
-      "bound_ratio": 0.05,
-      "stochastic_rounding": true,
-      "master_weights": "none",
-      "fused_single_pass": true,
-      "ema_decay": "auto",
-      "fsdp_sync": false,
-      "description": "Single-Buffer Triangle-Defect Convex Lookahead Momentum (TD-CLM) using the Triangle Inequality Envelope (cuts optimizer state RAM by 50% with zero global reductions in coordinate mode)."
-    },
-    "radial_trust_region_bound": {
-      "buffer": 2,
-      "tau": 0.8,
-      "cautious": true,
-      "cautious_clamp_min": 0.5,
-      "bound": true,
-      "bound_type": "radial",
-      "bound_ratio": 0.03,
-      "stochastic_rounding": true,
-      "master_weights": "none",
-      "fused_single_pass": true,
-      "ema_decay": "auto",
-      "fsdp_sync": false,
-      "description": "Direction-preserving radial trust-region bounding (tanh(r)/r) preserving 100% update angular geometry."
-    },
-    "semi_hybrid_fp32_master": {
-      "buffer": 2,
-      "cautious": true,
-      "cautious_clamp_min": 0.5,
-      "bound": true,
-      "bound_type": "coordinate",
-      "bound_ratio": 0.05,
-      "stochastic_rounding": true,
-      "master_weights": "semi",
-      "fused_single_pass": true,
-      "fsdp_sync": false,
-      "description": "Hybrid precision pairing 32-bit FP32 master weights with 16-bit BF16/FP16 optimizer moment buffers."
-    },
-    "full_fp32_baseline": {
-      "buffer": 2,
-      "cautious": true,
-      "cautious_clamp_min": 0.5,
-      "bound": true,
-      "bound_type": "coordinate",
-      "bound_ratio": 0.05,
-      "stochastic_rounding": true,
-      "master_weights": "full",
-      "fused_single_pass": true,
-      "fsdp_sync": false,
-      "description": "Full-precision FP32 master weights and FP32 optimizer states for controlled baseline evaluation."
-    },
-    "fsdp_distributed_sync": {
-      "buffer": 2,
-      "cautious": true,
-      "cautious_clamp_min": 0.5,
-      "bound": true,
-      "bound_type": "radial",
-      "bound_ratio": 0.03,
-      "stochastic_rounding": true,
-      "master_weights": "none",
-      "fused_single_pass": true,
-      "fsdp_sync": true,
-      "description": "Distributed multi-GPU training mode under PyTorch FSDP or DeepSpeed ZeRO-3 with cross-rank collective scratch synchronization."
-    }
-  }
-}
+{
+  "name": "LuminaV",
+  "version": "1.6.0",
+  "package": "luminamoon",
+  "package_version": "1.0.9",
+  "description": "Master-free, low-precision BF16/FP16 deep learning optimizer with Dual-Buffer Convex Lookahead Momentum (CLM), Single-Buffer Triangle-Defect Convex Lookahead Momentum (TD-CLM) via the Triangle Inequality Envelope, rational Pade [13/6] bounding, stateless 32-bit MurmurHash3 stochastic rounding, and native Rust SIMD / Vulkan / Triton execution.",
+  "repository": "https://github.com/cloverx-id/luminamoon",
+  "documentation": "https://cloverx-id.github.io/luminamoon/",
+  "default_params": {
+    "lr": 0.0008,
+    "betas": [
+      0.9,
+      0.999
+    ],
+    "eps": 1e-08,
+    "weight_decay": 0.08,
+    "tau": 0.8,
+    "lam": 0.3,
+    "cautious": true,
+    "cautious_clamp_min": 0.5,
+    "buffer": 2,
+    "stochastic_rounding": true,
+    "bound": true,
+    "bound_type": "coordinate",
+    "bound_ratio": 0.05,
+    "master_weights": "none",
+    "execution": "auto",
+    "fused_single_pass": true,
+    "fuse_tensors": false,
+    "bucket_cap_mb": 500.0,
+    "prewarm_jit": true,
+    "ema_decay": "auto",
+    "fsdp_sync": false,
+    "seed": 1337
+  },
+  "presets": {
+    "dual_buffer_clm": {
+      "buffer": 2,
+      "tau": 0.8,
+      "cautious": true,
+      "cautious_clamp_min": 0.5,
+      "bound": true,
+      "bound_type": "coordinate",
+      "bound_ratio": 0.05,
+      "stochastic_rounding": true,
+      "master_weights": "none",
+      "fused_single_pass": true,
+      "ema_decay": "auto",
+      "fsdp_sync": false,
+      "description": "Default Dual-Buffer Convex Lookahead Momentum (CLM) tracking m_t and innovation variance v_t with coordinate trust-region bounding."
+    },
+    "single_buffer_td_clm": {
+      "buffer": 1,
+      "lam": 0.3,
+      "cautious": true,
+      "cautious_clamp_min": 0.5,
+      "bound": true,
+      "bound_type": "coordinate",
+      "bound_ratio": 0.05,
+      "stochastic_rounding": true,
+      "master_weights": "none",
+      "fused_single_pass": true,
+      "ema_decay": "auto",
+      "fsdp_sync": false,
+      "description": "Single-Buffer Triangle-Defect Convex Lookahead Momentum (TD-CLM) using the Triangle Inequality Envelope (cuts optimizer state RAM by 50% with zero global reductions in coordinate mode)."
+    },
+    "radial_trust_region_bound": {
+      "buffer": 2,
+      "tau": 0.8,
+      "cautious": true,
+      "cautious_clamp_min": 0.5,
+      "bound": true,
+      "bound_type": "radial",
+      "bound_ratio": 0.03,
+      "stochastic_rounding": true,
+      "master_weights": "none",
+      "fused_single_pass": true,
+      "ema_decay": "auto",
+      "fsdp_sync": false,
+      "description": "Direction-preserving radial trust-region bounding (tanh(r)/r) preserving 100% update angular geometry."
+    },
+    "semi_hybrid_fp32_master": {
+      "buffer": 2,
+      "cautious": true,
+      "cautious_clamp_min": 0.5,
+      "bound": true,
+      "bound_type": "coordinate",
+      "bound_ratio": 0.05,
+      "stochastic_rounding": true,
+      "master_weights": "semi",
+      "fused_single_pass": true,
+      "fsdp_sync": false,
+      "description": "Hybrid precision pairing 32-bit FP32 master weights with 16-bit BF16/FP16 optimizer moment buffers."
+    },
+    "full_fp32_baseline": {
+      "buffer": 2,
+      "cautious": true,
+      "cautious_clamp_min": 0.5,
+      "bound": true,
+      "bound_type": "coordinate",
+      "bound_ratio": 0.05,
+      "stochastic_rounding": true,
+      "master_weights": "full",
+      "fused_single_pass": true,
+      "fsdp_sync": false,
+      "description": "Full-precision FP32 master weights and FP32 optimizer states for controlled baseline evaluation."
+    },
+    "fsdp_distributed_sync": {
+      "buffer": 2,
+      "cautious": true,
+      "cautious_clamp_min": 0.5,
+      "bound": true,
+      "bound_type": "radial",
+      "bound_ratio": 0.03,
+      "stochastic_rounding": true,
+      "master_weights": "none",
+      "fused_single_pass": true,
+      "fsdp_sync": true,
+      "description": "Distributed multi-GPU training mode under PyTorch FSDP or DeepSpeed ZeRO-3 with cross-rank collective scratch synchronization."
+    }
+  }
+}

data/Cargo.toml

@@ -1,24 +1,24 @@
-[package]
-name = "luminamoon-dataloader"
-version.workspace = true
-edition.workspace = true
-license.workspace = true
-
-[lib]
-name = "luminamoon_dataloader"
-path = "src/lib.rs"
-
-[dependencies]
-pyo3.workspace = true
-thiserror.workspace = true
-serde.workspace = true
-serde_json.workspace = true
-rayon.workspace = true
-parquet.workspace = true
-arrow-array.workspace = true
-arrow-schema.workspace = true
-arrow-ipc.workspace = true
-xxhash-rust.workspace = true
-crc32fast.workspace = true
-hashbrown.workspace = true
-parking_lot.workspace = true
+[package]
+name = "luminamoon-dataloader"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+
+[lib]
+name = "luminamoon_dataloader"
+path = "src/lib.rs"
+
+[dependencies]
+pyo3.workspace = true
+thiserror.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+rayon.workspace = true
+parquet.workspace = true
+arrow-array.workspace = true
+arrow-schema.workspace = true
+arrow-ipc.workspace = true
+xxhash-rust.workspace = true
+crc32fast.workspace = true
+hashbrown.workspace = true
+parking_lot.workspace = true

data/README.md

@@ -1,206 +1,206 @@
-# LuminaMoon Data Engine v1.1.0
-
-LuminaMoon Data (`luminamoon.data`) provides text cleaning, SIMD normalization, heuristic quality filtering, exact and fuzzy deduplication, and crash-resilient checkpointing for Python and Rust data pipelines, including PyTorch, Hugging Face datasets, Ray, and WebDataset.
-
----
-
-## 1. Modular Architecture & Responsibilities
-
-LuminaMoon Data focuses on in-memory text processing and deterministic state tracking, integrating smoothly with existing dataset readers:
-
-```mermaid
-flowchart LR
-    subgraph External ["External Data Sources"]
-        HF["Hugging Face datasets.load_dataset(streaming=True)"]
-        Torch["PyTorch torch.utils.data.DataLoader"]
-        Custom["Custom Python Generator / Iterables"]
-    end
-
-    subgraph Data_Engine ["LuminaMoon Data Engine (Rust Core)"]
-        direction TB
-        Cleaner["1. TextCleaner\n(AVX-512/AVX2/NEON SIMD CRLF & JSON path extraction)"]
-        Quality["2. QualityFilter\n(Shannon entropy, repetition loop, symbol ratio)"]
-        Dedup["3. StreamDeduplicator & MinHash\n(XXH3-64, SwissTable WAL, MinHash fuzzy LSH)"]
-        Masker["4. PIIMasker\n(Email, IP, Bearer/API Key, Card pattern sanitizer)"]
-        Tracker["5. StateTracker\n(Dual-Slot 128B Ping-Pong Slot A/B CRC32)"]
-        Glue["6. Collate & Pipeline Glue\n(PyTorch collate_fn, ResumablePipeline)"]
-    end
-
-    HF --> Cleaner
-    Torch --> Cleaner
-    Custom --> Cleaner
-    Cleaner --> Quality
-    Quality --> Dedup
-    Dedup --> Masker
-    Masker --> Tracker
-    Tracker --> Glue
-    Glue --> Consumer["GPU Model Training Step"]
-```
-
----
-
-## 2. Component Breakdown
-
-| Module | Core Responsibility | Underlying Implementation |
-| :--- | :--- | :--- |
-| **`luminamoon.data.cleaner`** | Whitespace trimming, SIMD newline normalization (`\r\n` $\rightarrow$ `\n`), byte length boundaries, and JSON path extraction. | `simd_cpu.rs`, `postprocess.rs` (Rayon parallel multi-threading, AVX-512BW, AVX2, NEON) |
-| **`luminamoon.data.quality`** | Shannon Character Entropy, loop repetition detector, and symbol-to-word ratio filtering. | `quality.rs` (Parallel Rayon heuristic quality filter) |
-| **`luminamoon.data.dedup`** | Stateful 64-bit exact hashing and deduplication with write-ahead log (`.dedup`) persistence. | `dedup.rs` (`xxh3_64` + `IdentityBuildHasher` SwissTable) |
-| **`luminamoon.data.minhash`** | Fuzzy near-duplicate detection via MinHash shingling and Jaccard similarity estimation. | `minhash.rs` (Permuted XXH3-64 MinHash signatures) |
-| **`luminamoon.data.masker`** | Pattern-based PII masking for emails, IPs, API keys/tokens, and payment cards. | `masker.py` (Compiled multi-pattern regex sanitizer) |
-| **`luminamoon.data.checkpoint`** | Dual-slot atomic coordinate checkpointing (`shard_id`, `row_offset`, `byte_offset`, `global_idx`). | `metadata.rs`, `coord.rs` (128-byte Dual-Slot Ping-Pong Slot A/B CRC32) |
-| **`luminamoon.data.string_tensor`** | Columnar string representation with CPU inlined StringView and GPU contiguous offset/payload buffers. | `string_tensor.rs` (16-byte Umbra/DuckDB StringView, SIMD prefix filter, contiguous u32 offsets + u8 payload) |
-| **`luminamoon.data.adapters`** | PyTorch `collate_fn` adapter and batch `ResumablePipeline`. | `adapters.py` (Dynamic token padding & tensor generation) |
-| **`luminamoon.data.serialization`** | Conversion of Python dictionaries to Parquet, JSONL, or Arrow IPC bytes. | `serialize.rs` (Arrow RecordBatch & Parquet ArrowWriter with ZSTD) |
-
----
-
-## 3. Quickstart & Usage
-
-### 3.1. Text Cleaning with SIMD & Rayon
-```python
-from luminamoon.data import TextCleaner
-
-cleaner = TextCleaner(
-    trim_whitespace=True,
-    normalize_newlines=True,      # AVX-512 / AVX2 / NEON SIMD
-    min_byte_len=10,
-    max_byte_len=2048,
-    truncate_over_max=True,
-    json_extract_key="data.text",  # Extracts dotted path in Rust
-)
-
-cleaned = cleaner.clean_batch(["  sample 1\r\n", "sample 2 "])
-```
-
-### 3.2. Quality Filtering (Shannon Entropy & Repetition Detector)
-```python
-from luminamoon.data import QualityFilter
-
-qf = QualityFilter(
-    min_entropy=2.5,             # Filters low-entropy noise
-    max_repetition_ratio=0.35,   # Filters repetitive text loops
-    max_symbol_ratio=0.30,       # Filters symbol-heavy dumps
-)
-
-if qf.is_high_quality("Sample prose text for language model training."):
-    ...
-```
-
-### 3.3. Exact & Fuzzy Near-Deduplication
-```python
-from luminamoon.data import StreamDeduplicator, MinHashDeduplicator
-
-# Exact dedup with WAL
-dedup = StreamDeduplicator(wal_path="./cache/dataset.dedup")
-unique_batch = dedup.filter_batch(cleaned)
-dedup.commit()
-
-# Fuzzy near-dedup (> 85% similarity)
-fuzzy = MinHashDeduplicator(threshold=0.85)
-if fuzzy.is_unique("A document with slight punctuation differences."):
-    ...
-```
-
-### 3.4. PII Sanitization
-```python
-from luminamoon.data import PIIMasker
-
-masker = PIIMasker(mask_emails=True, mask_ips=True, mask_api_keys=True)
-safe_text = masker.mask("Contact support@lumi.ai with key Bearer sk-1234567890abcdef1234567890abcdef")
-# Result: "Contact [EMAIL] with key [API_KEY]"
-```
-
-### 3.5. Dual-Layout StringTensor (CPU StringView & GPU Buffers)
-```python
-from luminamoon.data import StringTensor
-
-# 1. CPU Layout (Pattern 2 - Umbra / DuckDB / Arrow 2.0 StringView)
-# Strings <= 12 bytes are stored inline in a 16-byte struct
-st = StringTensor(["short", "this is a longer sample exceeding 12 bytes", "pref_sample"])
-print(st.is_inlined_mask())  # [True, False, True]
-
-# SIMD prefix and length filtering
-filtered = st.filter_by_prefix("pref_")
-hashes = st.hash_all()  # Multi-threaded XXH3-64 hashing in Rust
-
-# 2. GPU Layout (Pattern 1 - Apache Arrow & TensorFlow Contiguous Offsets + Data Buffer)
-offsets, payload = st.to_gpu_buffers()      # Raw uint32 offsets and uint8 payload
-off_tensor, pay_tensor = st.to_torch_gpu()  # PyTorch tensors
-off_arr, pay_arr = st.to_numpy_gpu()        # NumPy buffers
-arrow_array = st.to_arrow()                 # Apache Arrow 2.0 StringArray
-polars_series = st.to_polars()              # Polars Series
-```
-
-### 3.6. Fused Single-Pass Pipeline
-```python
-from luminamoon.data import FusedPipeline, StreamDeduplicator
-
-pipeline = FusedPipeline(
-    trim_whitespace=True,
-    normalize_newlines=True,
-    min_chars=15,
-    min_entropy=2.0,
-    deduplicator=StreamDeduplicator(),
-    commit_dedup=True,
-)
-
-# Process a batch in a single multi-threaded Rayon pass
-string_tensor = pipeline(raw_records)
-```
-
-### 3.7. PyTorch Collate & Async Prefetch Pipeline
-```python
-from torch.utils.data import DataLoader
-from luminamoon.data import create_resumable_collate, ResumablePipeline, FusedPipeline
-
-# 1. As PyTorch collate_fn
-collate_fn = create_resumable_collate(
-    fused_pipeline=FusedPipeline(),
-    return_string_tensor=True,
-    tokenizer=my_tokenizer,
-    max_length=512,
-)
-loader = DataLoader(raw_dataset, batch_size=64, collate_fn=collate_fn)
-
-# 2. Or as batch ResumablePipeline with background prefetcher
-resumable_pipe = ResumablePipeline(
-    source=raw_dataset,
-    fused_pipeline=FusedPipeline(),
-    batch_size=64,
-    prefetch_batches=4,  # Background prefetch worker for processed batches
-    return_string_tensor=True,
-)
-for batch in resumable_pipe:
-    ...
-```
-
----
-
-## 4. Verification & Testing
-
-Run Rust tests:
-
-```bash
-cargo test -p luminamoon-dataloader
-```
-
-Run Python modular test suite:
-
-```bash
-python -B tests/test_data_modular.py
-```
-
----
-
-## 5. Version Inspection
-
-Query the current data engine module version:
-
-```python
-import luminamoon.data as lmd
-
-print(lmd.get_version())  # Active package release version string
-```
-
-> **Note**: LuminaMoon Data operates on a single unified native SIMD engine. Package releases follow `luminamoon.__version__`.
+# LuminaMoon Data Engine v1.1.0
+
+LuminaMoon Data (`luminamoon.data`) provides text cleaning, SIMD normalization, heuristic quality filtering, exact and fuzzy deduplication, and crash-resilient checkpointing for Python and Rust data pipelines, including PyTorch, Hugging Face datasets, Ray, and WebDataset.
+
+---
+
+## 1. Modular Architecture & Responsibilities
+
+LuminaMoon Data focuses on in-memory text processing and deterministic state tracking, integrating smoothly with existing dataset readers:
+
+```mermaid
+flowchart LR
+    subgraph External ["External Data Sources"]
+        HF["Hugging Face datasets.load_dataset(streaming=True)"]
+        Torch["PyTorch torch.utils.data.DataLoader"]
+        Custom["Custom Python Generator / Iterables"]
+    end
+
+    subgraph Data_Engine ["LuminaMoon Data Engine (Rust Core)"]
+        direction TB
+        Cleaner["1. TextCleaner\n(AVX-512/AVX2/NEON SIMD CRLF & JSON path extraction)"]
+        Quality["2. QualityFilter\n(Shannon entropy, repetition loop, symbol ratio)"]
+        Dedup["3. StreamDeduplicator & MinHash\n(XXH3-64, SwissTable WAL, MinHash fuzzy LSH)"]
+        Masker["4. PIIMasker\n(Email, IP, Bearer/API Key, Card pattern sanitizer)"]
+        Tracker["5. StateTracker\n(Dual-Slot 128B Ping-Pong Slot A/B CRC32)"]
+        Glue["6. Collate & Pipeline Glue\n(PyTorch collate_fn, ResumablePipeline)"]
+    end
+
+    HF --> Cleaner
+    Torch --> Cleaner
+    Custom --> Cleaner
+    Cleaner --> Quality
+    Quality --> Dedup
+    Dedup --> Masker
+    Masker --> Tracker
+    Tracker --> Glue
+    Glue --> Consumer["GPU Model Training Step"]
+```
+
+---
+
+## 2. Component Breakdown
+
+| Module | Core Responsibility | Underlying Implementation |
+| :--- | :--- | :--- |
+| **`luminamoon.data.cleaner`** | Whitespace trimming, SIMD newline normalization (`\r\n` $\rightarrow$ `\n`), byte length boundaries, and JSON path extraction. | `simd_cpu.rs`, `postprocess.rs` (Rayon parallel multi-threading, AVX-512BW, AVX2, NEON) |
+| **`luminamoon.data.quality`** | Shannon Character Entropy, loop repetition detector, and symbol-to-word ratio filtering. | `quality.rs` (Parallel Rayon heuristic quality filter) |
+| **`luminamoon.data.dedup`** | Stateful 64-bit exact hashing and deduplication with write-ahead log (`.dedup`) persistence. | `dedup.rs` (`xxh3_64` + `IdentityBuildHasher` SwissTable) |
+| **`luminamoon.data.minhash`** | Fuzzy near-duplicate detection via MinHash shingling and Jaccard similarity estimation. | `minhash.rs` (Permuted XXH3-64 MinHash signatures) |
+| **`luminamoon.data.masker`** | Pattern-based PII masking for emails, IPs, API keys/tokens, and payment cards. | `masker.py` (Compiled multi-pattern regex sanitizer) |
+| **`luminamoon.data.checkpoint`** | Dual-slot atomic coordinate checkpointing (`shard_id`, `row_offset`, `byte_offset`, `global_idx`). | `metadata.rs`, `coord.rs` (128-byte Dual-Slot Ping-Pong Slot A/B CRC32) |
+| **`luminamoon.data.string_tensor`** | Columnar string representation with CPU inlined StringView and GPU contiguous offset/payload buffers. | `string_tensor.rs` (16-byte Umbra/DuckDB StringView, SIMD prefix filter, contiguous u32 offsets + u8 payload) |
+| **`luminamoon.data.adapters`** | PyTorch `collate_fn` adapter and batch `ResumablePipeline`. | `adapters.py` (Dynamic token padding & tensor generation) |
+| **`luminamoon.data.serialization`** | Conversion of Python dictionaries to Parquet, JSONL, or Arrow IPC bytes. | `serialize.rs` (Arrow RecordBatch & Parquet ArrowWriter with ZSTD) |
+
+---
+
+## 3. Quickstart & Usage
+
+### 3.1. Text Cleaning with SIMD & Rayon
+```python
+from luminamoon.data import TextCleaner
+
+cleaner = TextCleaner(
+    trim_whitespace=True,
+    normalize_newlines=True,      # AVX-512 / AVX2 / NEON SIMD
+    min_byte_len=10,
+    max_byte_len=2048,
+    truncate_over_max=True,
+    json_extract_key="data.text",  # Extracts dotted path in Rust
+)
+
+cleaned = cleaner.clean_batch(["  sample 1\r\n", "sample 2 "])
+```
+
+### 3.2. Quality Filtering (Shannon Entropy & Repetition Detector)
+```python
+from luminamoon.data import QualityFilter
+
+qf = QualityFilter(
+    min_entropy=2.5,             # Filters low-entropy noise
+    max_repetition_ratio=0.35,   # Filters repetitive text loops
+    max_symbol_ratio=0.30,       # Filters symbol-heavy dumps
+)
+
+if qf.is_high_quality("Sample prose text for language model training."):
+    ...
+```
+
+### 3.3. Exact & Fuzzy Near-Deduplication
+```python
+from luminamoon.data import StreamDeduplicator, MinHashDeduplicator
+
+# Exact dedup with WAL
+dedup = StreamDeduplicator(wal_path="./cache/dataset.dedup")
+unique_batch = dedup.filter_batch(cleaned)
+dedup.commit()
+
+# Fuzzy near-dedup (> 85% similarity)
+fuzzy = MinHashDeduplicator(threshold=0.85)
+if fuzzy.is_unique("A document with slight punctuation differences."):
+    ...
+```
+
+### 3.4. PII Sanitization
+```python
+from luminamoon.data import PIIMasker
+
+masker = PIIMasker(mask_emails=True, mask_ips=True, mask_api_keys=True)
+safe_text = masker.mask("Contact support@lumi.ai with key Bearer sk-1234567890abcdef1234567890abcdef")
+# Result: "Contact [EMAIL] with key [API_KEY]"
+```
+
+### 3.5. Dual-Layout StringTensor (CPU StringView & GPU Buffers)
+```python
+from luminamoon.data import StringTensor
+
+# 1. CPU Layout (Pattern 2 - Umbra / DuckDB / Arrow 2.0 StringView)
+# Strings <= 12 bytes are stored inline in a 16-byte struct
+st = StringTensor(["short", "this is a longer sample exceeding 12 bytes", "pref_sample"])
+print(st.is_inlined_mask())  # [True, False, True]
+
+# SIMD prefix and length filtering
+filtered = st.filter_by_prefix("pref_")
+hashes = st.hash_all()  # Multi-threaded XXH3-64 hashing in Rust
+
+# 2. GPU Layout (Pattern 1 - Apache Arrow & TensorFlow Contiguous Offsets + Data Buffer)
+offsets, payload = st.to_gpu_buffers()      # Raw uint32 offsets and uint8 payload
+off_tensor, pay_tensor = st.to_torch_gpu()  # PyTorch tensors
+off_arr, pay_arr = st.to_numpy_gpu()        # NumPy buffers
+arrow_array = st.to_arrow()                 # Apache Arrow 2.0 StringArray
+polars_series = st.to_polars()              # Polars Series
+```
+
+### 3.6. Fused Single-Pass Pipeline
+```python
+from luminamoon.data import FusedPipeline, StreamDeduplicator
+
+pipeline = FusedPipeline(
+    trim_whitespace=True,
+    normalize_newlines=True,
+    min_chars=15,
+    min_entropy=2.0,
+    deduplicator=StreamDeduplicator(),
+    commit_dedup=True,
+)
+
+# Process a batch in a single multi-threaded Rayon pass
+string_tensor = pipeline(raw_records)
+```
+
+### 3.7. PyTorch Collate & Async Prefetch Pipeline
+```python
+from torch.utils.data import DataLoader
+from luminamoon.data import create_resumable_collate, ResumablePipeline, FusedPipeline
+
+# 1. As PyTorch collate_fn
+collate_fn = create_resumable_collate(
+    fused_pipeline=FusedPipeline(),
+    return_string_tensor=True,
+    tokenizer=my_tokenizer,
+    max_length=512,
+)
+loader = DataLoader(raw_dataset, batch_size=64, collate_fn=collate_fn)
+
+# 2. Or as batch ResumablePipeline with background prefetcher
+resumable_pipe = ResumablePipeline(
+    source=raw_dataset,
+    fused_pipeline=FusedPipeline(),
+    batch_size=64,
+    prefetch_batches=4,  # Background prefetch worker for processed batches
+    return_string_tensor=True,
+)
+for batch in resumable_pipe:
+    ...
+```
+
+---
+
+## 4. Verification & Testing
+
+Run Rust tests:
+
+```bash
+cargo test -p luminamoon-dataloader
+```
+
+Run Python modular test suite:
+
+```bash
+python -B tests/test_data_modular.py
+```
+
+---
+
+## 5. Version Inspection
+
+Query the current data engine module version:
+
+```python
+import luminamoon.data as lmd
+
+print(lmd.get_version())  # Active package release version string
+```
+
+> **Note**: LuminaMoon Data operates on a single unified native SIMD engine. Package releases follow `luminamoon.__version__`.

data/docs/README.md

@@ -1,62 +1,62 @@
-# LuminaMoon Data Engine Documentation
-
-Documentation index for the **LuminaMoon Data Engine (`luminamoon.data` / `luminamoon-dataloader`)**.
-
----
-
-## 1. Architecture, User Guide & Developer Specs
-
-- **[LuminaMoon Data Engine Guide (`data/README.md`)](../README.md)**: Architecture overview, crate module breakdown, `FusedPipeline` single-pass Rayon execution, dual-layout `StringTensor` (CPU 16-byte `StringView16` & GPU contiguous buffers), 128-byte Ping-Pong checkpoint recovery (`Slot A` / `Slot B`), two-stage `xxh3_64` deduplication, 64-permutation MinHash LSH, `AVX-512BW` / `AVX2` / `NEON` CPU SIMD normalization, and PyTorch `ResumablePipeline` / `create_resumable_collate` integration.
-- **[Developer Modular Documentation Index (`docs/dataloader/README.md`)](../../docs/dataloader/README.md)**:
-  - **[01. Coordinates & Dual Metadata (`coord.rs`, `metadata.rs`, `constants.rs`, `error.rs`)](../../docs/dataloader/coord_and_metadata/README.md)**
-  - **[02. Two-Stage XXH3-64 Deduplication (`dedup.rs`)](../../docs/dataloader/dedup/README.md)**
-  - **[03. CPU SIMD & Parallel Post-Processing (`simd_cpu.rs`, `postprocess.rs`)](../../docs/dataloader/simd_and_postprocess/README.md)**
-  - **[04. Heuristic Quality Filter & MinHash LSH (`quality.rs`, `minhash.rs`)](../../docs/dataloader/sources_and_streaming/README.md)**
-  - **[05. Dual-Layout `StringTensor` & Resumable Adapters (`string_tensor.rs`, `adapters.py`)](../../docs/dataloader/preram/README.md)**
-  - **[06. Columnar Table Serialization (`serialize.rs`, `serialization.py`)](../../docs/dataloader/multimodal_and_hf_hub/README.md)**
-  - **[07. PyO3 `abi3t` FFI Bridge & Modular Runtime (`py_bridge.rs`, `python/luminamoon/data/`)](../../docs/dataloader/python_ffi_and_runtime/README.md)**
-
----
-
-## 2. External Library Compatibility Notes (`data/docs/library_audit/`)
-
-Version verification, changelog tracking, and compatibility notes for external dependencies used by `luminamoon-dataloader`:
-
-- **[Library Compatibility Index](library_audit/README.md)**
-- **[01. Rust Toolchain, SIMD & Memory Alignment (`rayon`)](library_audit/01_rust_toolchain_simd_and_mmap.md)**: Rust 1.99.1, Edition 2024 `unsafe_op_in_unsafe_fn`, `OnceLock` ISA caching, and `AVX-512BW` / `AVX2` / `NEON` intrinsics.
-- **[02. Python, PyO3 (`abi3t`), and Maturin](library_audit/02_python_pyo3_abi3t_and_maturin.md)**: Free-threaded Python (`abi3t`), `py.detach` GIL release, `Bound::cast::<T>()`, `PyBool` vs `PyInt` type ordering, and table/postprocess FFI functions.
-- **[03. Apache Arrow & Parquet (`arrow-array`, `arrow-schema`, `parquet`)](library_audit/03_parquet_arrow_and_http206_streaming.md)**: Columnar `RecordBatch` construction, ZSTD Parquet compression, and HuggingFace Dataset Viewer `Struct<bytes, path>` schema generation.
-- **[04. `xxhash-rust`, `hashbrown`, `crc32fast`, and `parking_lot`](library_audit/04_xxh3_dedup_crc32_and_hf_hub_security.md)**: `xxh3_64` + `IdentityBuildHasher` SwissTable, `crc32fast` Ping-Pong checkpoint slots, and synchronization primitives.
-- **[05. Universal Formats (`arrow-ipc`, `serde_json`) and Typo Diagnostics](library_audit/05_universal_formats_compression_and_diagnostics.md)**: Arrow IPC (`ARROW1`), NDJSON serialization, and Damerau-Levenshtein parameter suggestions (`did_you_mean`).
-
----
-
-## 3. Module Summary & Verification
-
-### 3.1. Module Matrix
-
-| Module / File | Core Responsibility | Status |
-| :--- | :--- | :--- |
-| [`data/src/constants.rs`](../src/constants.rs) | Magic bytes (`LMDL`, `PAR1`, `ARROW1`), slot sizes, and default bounds. | Verified |
-| [`data/src/coord.rs`](../src/coord.rs) | 32-byte little-endian `StreamCoordinate` (`shard_id`, `byte_offset`, `row_group_idx`, `page_idx`, `row_offset_in_page`, `global_row_idx`). | Verified |
-| [`data/src/dedup.rs`](../src/dedup.rs) | `TwoStageDeduplicator` using `xxh3_64` with `IdentityBuildHasher` SwissTable. | Verified |
-| [`data/src/error.rs`](../src/error.rs) | Unified `DataLoaderError` with Damerau-Levenshtein edit-distance suggestions (`did_you_mean`). | Verified |
-| [`data/src/metadata.rs`](../src/metadata.rs) | 128-byte Dual-Slot Ping-Pong (`Slot A` / `Slot B`) checkpoint file with IEEE CRC32 (`crc32fast`) and `.dedup` WAL. | Verified |
-| [`data/src/minhash.rs`](../src/minhash.rs) | 64-permutation MinHash LSH fuzzy deduplicator and Jaccard similarity estimation. | Verified |
-| [`data/src/quality.rs`](../src/quality.rs) | Shannon Character Entropy, loop repetition detector, and symbol-to-word ratio filters. | Verified |
-| [`data/src/simd_cpu.rs`](../src/simd_cpu.rs) | `AVX-512BW`, `AVX2`, `NEON`, and Scalar kernels (`find_byte_fast`, `rfind_byte_fast`) + 64-byte `CacheAlignedBuffer`. | Verified |
-| [`data/src/string_tensor.rs`](../src/string_tensor.rs) | Dual-layout container for CPU `StringView16` (<=12B inline) and GPU contiguous offsets/payload buffers. | Verified |
-| [`data/src/postprocess.rs`](../src/postprocess.rs) | Rayon parallel UTF-8 validation, SIMD `\r` detection, JSON dotted-path extraction, and whitespace normalization. | Verified |
-| [`data/src/serialize.rs`](../src/serialize.rs) | Table serializers for Apache Arrow IPC, Parquet (ZSTD), and NDJSON (`DataValue`). | Verified |
-| [`data/src/py_bridge.rs`](../src/py_bridge.rs) | `PyO3 0.30.1` (`abi3t`) bindings for `FusedPipeline`, `StringTensor`, `StateTracker`, `StreamDeduplicator`, `QualityFilter`, and `MinHashDeduplicator`. | Verified |
-| [`python/luminamoon/data/`](../../python/luminamoon/data/__init__.py) | Modular Python package (`fused.py`, `cleaner.py`, `quality.py`, `dedup.py`, `minhash.py`, `masker.py`, `string_tensor.py`, `checkpoint.py`, `adapters.py`, `serialization.py`, `version.py`, and `fallbacks/`). | Verified |
-
-### 3.2. Verification Commands
-
-| Check | Command | Result |
-| :--- | :--- | :--- |
-| Rust Workspace Linter | `cargo clippy --workspace --all-targets -- -D warnings` | **PASSED** (0 Warnings) |
-| Rust Release Tests | `cargo test --workspace` | **PASSED** |
-| Python Static Type Check | `npx pyright python/luminamoon` | **PASSED** (0 Errors, 0 Warnings) |
-| Python Modular Data Tests | `python -B tests/test_data_modular.py` | **PASSED** |
-| Python StringTensor CPU/GPU Tests | `python -B tests/test_gpu_vs_cpu_strings.py` | **PASSED** |
+# LuminaMoon Data Engine Documentation
+
+Documentation index for the **LuminaMoon Data Engine (`luminamoon.data` / `luminamoon-dataloader`)**.
+
+---
+
+## 1. Architecture, User Guide & Developer Specs
+
+- **[LuminaMoon Data Engine Guide (`data/README.md`)](../README.md)**: Architecture overview, crate module breakdown, `FusedPipeline` single-pass Rayon execution, dual-layout `StringTensor` (CPU 16-byte `StringView16` & GPU contiguous buffers), 128-byte Ping-Pong checkpoint recovery (`Slot A` / `Slot B`), two-stage `xxh3_64` deduplication, 64-permutation MinHash LSH, `AVX-512BW` / `AVX2` / `NEON` CPU SIMD normalization, and PyTorch `ResumablePipeline` / `create_resumable_collate` integration.
+- **[Developer Modular Documentation Index (`docs/dataloader/README.md`)](../../docs/dataloader/README.md)**:
+  - **[01. Coordinates & Dual Metadata (`coord.rs`, `metadata.rs`, `constants.rs`, `error.rs`)](../../docs/dataloader/coord_and_metadata/README.md)**
+  - **[02. Two-Stage XXH3-64 Deduplication (`dedup.rs`)](../../docs/dataloader/dedup/README.md)**
+  - **[03. CPU SIMD & Parallel Post-Processing (`simd_cpu.rs`, `postprocess.rs`)](../../docs/dataloader/simd_and_postprocess/README.md)**
+  - **[04. Heuristic Quality Filter & MinHash LSH (`quality.rs`, `minhash.rs`)](../../docs/dataloader/sources_and_streaming/README.md)**
+  - **[05. Dual-Layout `StringTensor` & Resumable Adapters (`string_tensor.rs`, `adapters.py`)](../../docs/dataloader/preram/README.md)**
+  - **[06. Columnar Table Serialization (`serialize.rs`, `serialization.py`)](../../docs/dataloader/multimodal_and_hf_hub/README.md)**
+  - **[07. PyO3 `abi3t` FFI Bridge & Modular Runtime (`py_bridge.rs`, `python/luminamoon/data/`)](../../docs/dataloader/python_ffi_and_runtime/README.md)**
+
+---
+
+## 2. External Library Compatibility Notes (`data/docs/library_audit/`)
+
+Version verification, changelog tracking, and compatibility notes for external dependencies used by `luminamoon-dataloader`:
+
+- **[Library Compatibility Index](library_audit/README.md)**
+- **[01. Rust Toolchain, SIMD & Memory Alignment (`rayon`)](library_audit/01_rust_toolchain_simd_and_mmap.md)**: Rust 1.99.1, Edition 2024 `unsafe_op_in_unsafe_fn`, `OnceLock` ISA caching, and `AVX-512BW` / `AVX2` / `NEON` intrinsics.
+- **[02. Python, PyO3 (`abi3t`), and Maturin](library_audit/02_python_pyo3_abi3t_and_maturin.md)**: Free-threaded Python (`abi3t`), `py.detach` GIL release, `Bound::cast::<T>()`, `PyBool` vs `PyInt` type ordering, and table/postprocess FFI functions.
+- **[03. Apache Arrow & Parquet (`arrow-array`, `arrow-schema`, `parquet`)](library_audit/03_parquet_arrow_and_http206_streaming.md)**: Columnar `RecordBatch` construction, ZSTD Parquet compression, and HuggingFace Dataset Viewer `Struct<bytes, path>` schema generation.
+- **[04. `xxhash-rust`, `hashbrown`, `crc32fast`, and `parking_lot`](library_audit/04_xxh3_dedup_crc32_and_hf_hub_security.md)**: `xxh3_64` + `IdentityBuildHasher` SwissTable, `crc32fast` Ping-Pong checkpoint slots, and synchronization primitives.
+- **[05. Universal Formats (`arrow-ipc`, `serde_json`) and Typo Diagnostics](library_audit/05_universal_formats_compression_and_diagnostics.md)**: Arrow IPC (`ARROW1`), NDJSON serialization, and Damerau-Levenshtein parameter suggestions (`did_you_mean`).
+
+---
+
+## 3. Module Summary & Verification
+
+### 3.1. Module Matrix
+
+| Module / File | Core Responsibility | Status |
+| :--- | :--- | :--- |
+| [`data/src/constants.rs`](../src/constants.rs) | Magic bytes (`LMDL`, `PAR1`, `ARROW1`), slot sizes, and default bounds. | Verified |
+| [`data/src/coord.rs`](../src/coord.rs) | 32-byte little-endian `StreamCoordinate` (`shard_id`, `byte_offset`, `row_group_idx`, `page_idx`, `row_offset_in_page`, `global_row_idx`). | Verified |
+| [`data/src/dedup.rs`](../src/dedup.rs) | `TwoStageDeduplicator` using `xxh3_64` with `IdentityBuildHasher` SwissTable. | Verified |
+| [`data/src/error.rs`](../src/error.rs) | Unified `DataLoaderError` with Damerau-Levenshtein edit-distance suggestions (`did_you_mean`). | Verified |
+| [`data/src/metadata.rs`](../src/metadata.rs) | 128-byte Dual-Slot Ping-Pong (`Slot A` / `Slot B`) checkpoint file with IEEE CRC32 (`crc32fast`) and `.dedup` WAL. | Verified |
+| [`data/src/minhash.rs`](../src/minhash.rs) | 64-permutation MinHash LSH fuzzy deduplicator and Jaccard similarity estimation. | Verified |
+| [`data/src/quality.rs`](../src/quality.rs) | Shannon Character Entropy, loop repetition detector, and symbol-to-word ratio filters. | Verified |
+| [`data/src/simd_cpu.rs`](../src/simd_cpu.rs) | `AVX-512BW`, `AVX2`, `NEON`, and Scalar kernels (`find_byte_fast`, `rfind_byte_fast`) + 64-byte `CacheAlignedBuffer`. | Verified |
+| [`data/src/string_tensor.rs`](../src/string_tensor.rs) | Dual-layout container for CPU `StringView16` (<=12B inline) and GPU contiguous offsets/payload buffers. | Verified |
+| [`data/src/postprocess.rs`](../src/postprocess.rs) | Rayon parallel UTF-8 validation, SIMD `\r` detection, JSON dotted-path extraction, and whitespace normalization. | Verified |
+| [`data/src/serialize.rs`](../src/serialize.rs) | Table serializers for Apache Arrow IPC, Parquet (ZSTD), and NDJSON (`DataValue`). | Verified |
+| [`data/src/py_bridge.rs`](../src/py_bridge.rs) | `PyO3 0.30.1` (`abi3t`) bindings for `FusedPipeline`, `StringTensor`, `StateTracker`, `StreamDeduplicator`, `QualityFilter`, and `MinHashDeduplicator`. | Verified |
+| [`python/luminamoon/data/`](../../python/luminamoon/data/__init__.py) | Modular Python package (`fused.py`, `cleaner.py`, `quality.py`, `dedup.py`, `minhash.py`, `masker.py`, `string_tensor.py`, `checkpoint.py`, `adapters.py`, `serialization.py`, `version.py`, and `fallbacks/`). | Verified |
+
+### 3.2. Verification Commands
+
+| Check | Command | Result |
+| :--- | :--- | :--- |
+| Rust Workspace Linter | `cargo clippy --workspace --all-targets -- -D warnings` | **PASSED** (0 Warnings) |
+| Rust Release Tests | `cargo test --workspace` | **PASSED** |
+| Python Static Type Check | `npx pyright python/luminamoon` | **PASSED** (0 Errors, 0 Warnings) |
+| Python Modular Data Tests | `python -B tests/test_data_modular.py` | **PASSED** |
+| Python StringTensor CPU/GPU Tests | `python -B tests/test_gpu_vs_cpu_strings.py` | **PASSED** |

data/docs/library_audit/01_rust_toolchain_simd_and_mmap.md

@@ -1,56 +1,56 @@
-# 1. Rust Toolchain, `std::arch` SIMD Intrinsics, `memmap2`, and Core Crates
-
-## 1.1 Latest Stable Versions
-- **Rust Compiler (`rustc` & `cargo`)**: `1.99.1` (`edition = "2024"`, `resolver = "3"`)
-- **`memmap2`**: `0.9.11`
-- **`rayon`**: `1.12.0`
-- **`thiserror`**: `2.0.21`
-- **`serde`**: `1.0.229`
-- **`serde_json`**: `1.0.154`
-
----
-
-## 1.2 Version Range Changelog (`Rust 1.85.0` - `1.99.1` & Supporting Crates)
-
-### A. Rust 1.85.0 (`edition = "2024"`)
-- **What changed**: The `unsafe_op_in_unsafe_fn` lint is enabled by default in Rust Edition 2024. Declaring a function as `unsafe fn` no longer implicitly permits calling `unsafe` operations inside the function body without an explicit `unsafe { ... }` block.
-- **Syntax changes**: Stabilized `let-chains` (`if let Some(a) = opt && a > 0`) inside `if` and `while` conditions, now enforced by `clippy::collapsible_if`.
-
-### B. Rust 1.86.0 - 1.89.0 (`std::arch` Target Features & `avx512bw` Stabilization)
-- **Rust 1.86.0 - 1.87.0**: Value-based SIMD intrinsics (such as `_mm512_set1_epi8`, `_mm512_cmpeq_epi8_mask`, `_mm256_set1_epi8`, `_mm256_cmpeq_epi8`, `_mm256_movemask_epi8`, `vdupq_n_u8`, `vceqq_u8`, `vmaxvq_u8`) no longer require an `unsafe` block when called inside a `#[target_feature]` function. Wrapping value intrinsics in `unsafe` triggers an `unused_unsafe` compiler warning. Only raw pointer dereference intrinsics (`_mm512_loadu_si512`, `_mm256_loadu_si256`, `vld1q_u8`) and pointer arithmetic (`ptr.add(offset)`) remain `unsafe`.
-- **Rust 1.89.0**: All `avx512f` and `avx512bw` target features and 512-bit byte intrinsics (`_mm512_cmpeq_epi8_mask`) were stabilized on the stable compiler channel without requiring `#![feature(stdarch_x86_avx512)]`.
-
-### C. Rust 1.88.0 - 1.95.0 (`[T]::as_chunks` & `std::cfg_select!`)
-- **Rust 1.88.0**: Stabilized the standard library method `slice::as_chunks::<N>()` to partition slices into fixed-size array chunks `&[[T; N]]` plus a remainder slice, replacing legacy `chunks_exact(N)` patterns (enforced by `clippy::chunks_exact_to_as_chunks`).
-- **Rust 1.95.0**: Added the standard macro `std::cfg_select!` for cross-architecture conditional compilation without requiring the external `cfg-if` crate.
-
-### D. `memmap2` (`0.9.0` - `0.9.11`): Mitigation of `RUSTSEC-2026-0186`
-- **What changed**: In `memmap2 < 0.9.11`, internal memory-mapping end-boundary calculations used `ptr.add(len)`. Under modern Rust/LLVM pointer provenance rules, calling `ptr.add` on an empty mapping or at the edge of the address space violates `inbounds` invariants and triggers Undefined Behavior (UB).
-- **Upstream fix (`0.9.11`)**: Replaced internal pointer boundary calculations with `wrapping_add` and stabilized `Advice::PopulateRead` (`MADV_POPULATE_READ` on Linux) for page pre-faulting.
-
-### E. `thiserror` (`1.0` - `2.0.21`), `rayon` (`1.10` - `1.12.0`), and `serde_json` (`1.0.120` - `1.0.154`)
-- **`thiserror 2.0.21`**: Migrated to the stabilized `core::error::Error` trait in `core`, removing mandatory reliance on internal `std::error::Error` paths and improving variable format expansion in `#[error("...")]` macros.
-- **`rayon 1.12.0`**: Improved work-stealing scheduling efficiency on indexed parallel iterators (`into_par_iter().enumerate()`).
-- **`serde_json 1.0.154`**: Optimized direct byte-buffer serialization (`serde_json::to_writer(&mut Vec<u8>, ...)`) without intermediate `String` allocations.
-
----
-
-## 1.3 Adaptations in LuminaMoon Source Code
-
-1. **Architecture Detection via `std::cfg_select!` and Scoped `unsafe` Intrinsic Blocks (`data/src/simd_cpu.rs`)**:
-   - Uses `std::cfg_select!` inside `detect_cpu_isa()` to select `x86`/`x86_64`, `aarch64`, or `Scalar` fallback branches without any `cfg-if` dependency.
-   - In `find_byte_avx512`, `rfind_byte_avx512`, `find_byte_avx2`, `rfind_byte_avx2`, `find_byte_neon`, and `rfind_byte_neon`, `unsafe { ... }` blocks are restricted strictly to `_mm512_loadu_si512`, `_mm256_loadu_si256`, `vld1q_u8`, and `ptr.add(...)`:
-     ```rust
-     while offset + 64 <= len {
-         let chunk = unsafe { _mm512_loadu_si512(ptr.add(offset).cast()) };
-         let mask = _mm512_cmpeq_epi8_mask(chunk, v_needle);
-         if mask != 0 {
-             return Some(offset + mask.trailing_zeros() as usize);
-         }
-         offset += 64;
-     }
-     ```
-2. **Adoption of `slice::as_chunks::<3>()` (`data/src/multimodal_serialize.rs`)**:
-   - The `encode_base64` function uses `let (chunks, rem) = input.as_chunks::<3>();` to process 3-byte blocks directly without per-iteration bounds checks.
-3. **Safe Memory Mapping with `memmap2 0.9.11` (`data/src/source_local.rs`)**:
-   - `LocalMmapLineSource::open` maps local files using `memmap2 0.9.11` (patched against `RUSTSEC-2026-0186`) and enforces `start_coord.byte_offset.min(total_len)` bounds checks prior to slicing.
+# 1. Rust Toolchain, `std::arch` SIMD Intrinsics, `memmap2`, and Core Crates
+
+## 1.1 Latest Stable Versions
+- **Rust Compiler (`rustc` & `cargo`)**: `1.99.1` (`edition = "2024"`, `resolver = "3"`)
+- **`memmap2`**: `0.9.11`
+- **`rayon`**: `1.12.0`
+- **`thiserror`**: `2.0.21`
+- **`serde`**: `1.0.229`
+- **`serde_json`**: `1.0.154`
+
+---
+
+## 1.2 Version Range Changelog (`Rust 1.85.0` - `1.99.1` & Supporting Crates)
+
+### A. Rust 1.85.0 (`edition = "2024"`)
+- **What changed**: The `unsafe_op_in_unsafe_fn` lint is enabled by default in Rust Edition 2024. Declaring a function as `unsafe fn` no longer implicitly permits calling `unsafe` operations inside the function body without an explicit `unsafe { ... }` block.
+- **Syntax changes**: Stabilized `let-chains` (`if let Some(a) = opt && a > 0`) inside `if` and `while` conditions, now enforced by `clippy::collapsible_if`.
+
+### B. Rust 1.86.0 - 1.89.0 (`std::arch` Target Features & `avx512bw` Stabilization)
+- **Rust 1.86.0 - 1.87.0**: Value-based SIMD intrinsics (such as `_mm512_set1_epi8`, `_mm512_cmpeq_epi8_mask`, `_mm256_set1_epi8`, `_mm256_cmpeq_epi8`, `_mm256_movemask_epi8`, `vdupq_n_u8`, `vceqq_u8`, `vmaxvq_u8`) no longer require an `unsafe` block when called inside a `#[target_feature]` function. Wrapping value intrinsics in `unsafe` triggers an `unused_unsafe` compiler warning. Only raw pointer dereference intrinsics (`_mm512_loadu_si512`, `_mm256_loadu_si256`, `vld1q_u8`) and pointer arithmetic (`ptr.add(offset)`) remain `unsafe`.
+- **Rust 1.89.0**: All `avx512f` and `avx512bw` target features and 512-bit byte intrinsics (`_mm512_cmpeq_epi8_mask`) were stabilized on the stable compiler channel without requiring `#![feature(stdarch_x86_avx512)]`.
+
+### C. Rust 1.88.0 - 1.95.0 (`[T]::as_chunks` & `std::cfg_select!`)
+- **Rust 1.88.0**: Stabilized the standard library method `slice::as_chunks::<N>()` to partition slices into fixed-size array chunks `&[[T; N]]` plus a remainder slice, replacing legacy `chunks_exact(N)` patterns (enforced by `clippy::chunks_exact_to_as_chunks`).
+- **Rust 1.95.0**: Added the standard macro `std::cfg_select!` for cross-architecture conditional compilation without requiring the external `cfg-if` crate.
+
+### D. `memmap2` (`0.9.0` - `0.9.11`): Mitigation of `RUSTSEC-2026-0186`
+- **What changed**: In `memmap2 < 0.9.11`, internal memory-mapping end-boundary calculations used `ptr.add(len)`. Under modern Rust/LLVM pointer provenance rules, calling `ptr.add` on an empty mapping or at the edge of the address space violates `inbounds` invariants and triggers Undefined Behavior (UB).
+- **Upstream fix (`0.9.11`)**: Replaced internal pointer boundary calculations with `wrapping_add` and stabilized `Advice::PopulateRead` (`MADV_POPULATE_READ` on Linux) for page pre-faulting.
+
+### E. `thiserror` (`1.0` - `2.0.21`), `rayon` (`1.10` - `1.12.0`), and `serde_json` (`1.0.120` - `1.0.154`)
+- **`thiserror 2.0.21`**: Migrated to the stabilized `core::error::Error` trait in `core`, removing mandatory reliance on internal `std::error::Error` paths and improving variable format expansion in `#[error("...")]` macros.
+- **`rayon 1.12.0`**: Improved work-stealing scheduling efficiency on indexed parallel iterators (`into_par_iter().enumerate()`).
+- **`serde_json 1.0.154`**: Optimized direct byte-buffer serialization (`serde_json::to_writer(&mut Vec<u8>, ...)`) without intermediate `String` allocations.
+
+---
+
+## 1.3 Adaptations in LuminaMoon Source Code
+
+1. **Architecture Detection via `std::cfg_select!` and Scoped `unsafe` Intrinsic Blocks (`data/src/simd_cpu.rs`)**:
+   - Uses `std::cfg_select!` inside `detect_cpu_isa()` to select `x86`/`x86_64`, `aarch64`, or `Scalar` fallback branches without any `cfg-if` dependency.
+   - In `find_byte_avx512`, `rfind_byte_avx512`, `find_byte_avx2`, `rfind_byte_avx2`, `find_byte_neon`, and `rfind_byte_neon`, `unsafe { ... }` blocks are restricted strictly to `_mm512_loadu_si512`, `_mm256_loadu_si256`, `vld1q_u8`, and `ptr.add(...)`:
+     ```rust
+     while offset + 64 <= len {
+         let chunk = unsafe { _mm512_loadu_si512(ptr.add(offset).cast()) };
+         let mask = _mm512_cmpeq_epi8_mask(chunk, v_needle);
+         if mask != 0 {
+             return Some(offset + mask.trailing_zeros() as usize);
+         }
+         offset += 64;
+     }
+     ```
+2. **Adoption of `slice::as_chunks::<3>()` (`data/src/multimodal_serialize.rs`)**:
+   - The `encode_base64` function uses `let (chunks, rem) = input.as_chunks::<3>();` to process 3-byte blocks directly without per-iteration bounds checks.
+3. **Safe Memory Mapping with `memmap2 0.9.11` (`data/src/source_local.rs`)**:
+   - `LocalMmapLineSource::open` maps local files using `memmap2 0.9.11` (patched against `RUSTSEC-2026-0186`) and enforces `start_coord.byte_offset.min(total_len)` bounds checks prior to slicing.

data/docs/library_audit/02_python_pyo3_abi3t_and_maturin.md

@@ -1,53 +1,53 @@
-# 2. Python, PyO3 (`abi3t`), and Maturin
-
-## 2.1 Latest Stable Versions
-- **Python**: `3.15.0` (supports both standard CPython and free-threaded `Py_GIL_DISABLED` builds)
-- **`pyo3`**: `0.30.1` (`features = ["abi3-py310", "abi3t"]`)
-- **`maturin`**: `1.15.0` (`requires = ["maturin>=1.15,<2.0"]`)
-
----
-
-## 2.2 Version Range Changelog (`pyo3 0.24.0` - `0.30.1` & `maturin 1.8` - `1.15.0`)
-
-### A. PyO3 0.25.0 - 0.26.0: Deprecation of `extension-module` Feature
-- **What changed**: The Cargo feature `pyo3/extension-module` was deprecated starting in `pyo3 0.26.0`.
-- **Upstream technical rationale**: Cargo features are additive across an entire workspace. Enabling `extension-module` in `Cargo.toml` caused `cargo test` to fail during linking because `libpython` symbols were intentionally omitted when that feature was active.
-- **New idiom**: Extension module linking is controlled directly by `maturin >= 1.8` (or the environment variable `PYO3_BUILD_EXTENSION_MODULE=1`), rather than static declarations in `Cargo.toml`.
-
-### B. PyO3 0.26.0 - 0.28.0: Removal of `Python::allow_threads` $\rightarrow$ `Python::detach`
-- **What changed**: `py.allow_threads(|| ...)` was deprecated in `pyo3 0.26.0` and **completely removed** in `pyo3 0.28.0`.
-- **Upstream technical rationale**: On free-threaded Python runtimes (PEP 703 / PEP 803 without the GIL), Python threads already execute in parallel; releasing the context actually detaches the OS thread from the CPython runtime state.
-- **New idiom**: Use `py.detach(|| ...)`.
-
-### C. PyO3 0.27.0 - 0.30.1: Removal of `Bound::downcast` $\rightarrow$ `Bound::cast::<T>()`
-- **What changed**: `Bound<'py, PyAny>::downcast::<T>()` and `downcast_into::<T>()` were deprecated and removed in `pyo3 0.30.1`, replaced by `Bound<'py, PyAny>::cast::<T>()` and `cast_into::<T>()`.
-- **Upstream technical rationale**: Unified smart-reference type conversion naming across the `Bound<'py, T>` API.
-
-### D. PyO3 0.29.0 - 0.30.1 & Python 3.15: Stable ABI `abi3t` (PEP 803) and `#[pyclass(frozen, subclass)]`
-- **What changed**:
-  1. Added the `abi3t` feature alongside `abi3-py310` to support the cross-version Stable ABI across Python 3.10–3.15 on both standard GIL builds and free-threaded builds (`python3.14t` / `python3.15t`).
-  2. Rust classes exposed to Python via `#[pyclass]` no longer permit Python-side subclassing by default unless the `subclass` attribute is explicitly specified, and `frozen` is recommended for lock-free concurrency without `PyCell` borrow-checking overhead in free-threaded mode.
-  3. Inside declarative `#[pymodule] mod ...` blocks, external sub-components are registered using functions annotated with `#[pymodule_init]`.
-
----
-
-## 2.3 Adaptations in LuminaMoon Source Code
-
-1. **`Cargo.toml` & `data/Cargo.toml`**:
-   - Configured `pyo3 = { version = "0.30.1", features = ["abi3-py310", "abi3t"] }` in `[workspace.dependencies]` without `"extension-module"`, allowing `cargo test -p luminamoon-dataloader` to link and run standalone.
-2. **Runtime Detachment via `py.detach` (`data/src/py_bridge.rs`)**:
-   - All disk I/O, `mmap` reads, HTTP 206 network streaming, Parquet schema inference, and HuggingFace Hub NDJSON uploads execute inside `py.detach(|| ...)`:
-     ```rust
-     let batch_opt = py
-         .detach(|| self.inner.next_batch())
-         .map_err(to_py_err)?;
-     ```
-3. **Type Casting via `Bound::cast::<T>()` and `PyBool` vs `PyInt` Hierarchy (`data/src/py_bridge.rs`)**:
-   - Uses `columns.cast::<PyDict>()`, `v.cast::<PySequence>()`, and `pair.cast::<PyTuple>()` inside `extract_py_columns_table`.
-   - Because `bool` is a subclass of `int` in CPython, `py_any_to_multimodal_value` always checks `item.is_instance_of::<PyBool>()` before `item.is_instance_of::<PyInt>()` so boolean columns are never coerced into `1`/`0` integers.
-4. **`#[pyclass(frozen, subclass)]` Declarations and `#[pymodule_init]` Registration (`data/src/py_bridge.rs` & `optimizer/src/lib.rs`)**:
-   - `PyDataStreamLoader`, `PyMultimodalParquetLoader`, and `PyHfHubUploader` are declared with `#[pyclass(frozen, subclass, ...)]` and synchronized on the Rust side using `parking_lot::Mutex`.
-   - `register_dataloader_classes(m)` is invoked from `#[pymodule_init] fn init(m: &Bound<'_, PyModule>)` inside `optimizer/src/lib.rs`, registering all classes and module-level functions (`serialize_table_to_bytes`, `postprocess_and_dedup_records`, `suggest_closest_matches`).
-5. **Automatic Row-to-Column Transposition & GIL-Free Table/Postprocess Functions (`data/src/py_bridge.rs`)**:
-   - `extract_py_columns_table` supports three Python input layouts simultaneously: `dict[str, Sequence[Any]]`, `Sequence[tuple[str, Sequence[Any]]]`, and `Sequence[dict[str, Any]]` (transposing row dictionaries into ordered columns by first-seen key order and filling missing keys with `MultimodalValue::Null`).
-   - `serialize_table_to_bytes` and `postprocess_and_dedup_records` (with `deduplicate: bool = true`) execute Parquet/Arrow IPC/CSV/JSONL/JSON serialization and Rayon + SIMD normalization + `xxh3_64` deduplication completely outside the GIL via `py.detach(|| ...)`.
+# 2. Python, PyO3 (`abi3t`), and Maturin
+
+## 2.1 Latest Stable Versions
+- **Python**: `3.15.0` (supports both standard CPython and free-threaded `Py_GIL_DISABLED` builds)
+- **`pyo3`**: `0.30.1` (`features = ["abi3-py310", "abi3t"]`)
+- **`maturin`**: `1.15.0` (`requires = ["maturin>=1.15,<2.0"]`)
+
+---
+
+## 2.2 Version Range Changelog (`pyo3 0.24.0` - `0.30.1` & `maturin 1.8` - `1.15.0`)
+
+### A. PyO3 0.25.0 - 0.26.0: Deprecation of `extension-module` Feature
+- **What changed**: The Cargo feature `pyo3/extension-module` was deprecated starting in `pyo3 0.26.0`.
+- **Upstream technical rationale**: Cargo features are additive across an entire workspace. Enabling `extension-module` in `Cargo.toml` caused `cargo test` to fail during linking because `libpython` symbols were intentionally omitted when that feature was active.
+- **New idiom**: Extension module linking is controlled directly by `maturin >= 1.8` (or the environment variable `PYO3_BUILD_EXTENSION_MODULE=1`), rather than static declarations in `Cargo.toml`.
+
+### B. PyO3 0.26.0 - 0.28.0: Removal of `Python::allow_threads` $\rightarrow$ `Python::detach`
+- **What changed**: `py.allow_threads(|| ...)` was deprecated in `pyo3 0.26.0` and **completely removed** in `pyo3 0.28.0`.
+- **Upstream technical rationale**: On free-threaded Python runtimes (PEP 703 / PEP 803 without the GIL), Python threads already execute in parallel; releasing the context actually detaches the OS thread from the CPython runtime state.
+- **New idiom**: Use `py.detach(|| ...)`.
+
+### C. PyO3 0.27.0 - 0.30.1: Removal of `Bound::downcast` $\rightarrow$ `Bound::cast::<T>()`
+- **What changed**: `Bound<'py, PyAny>::downcast::<T>()` and `downcast_into::<T>()` were deprecated and removed in `pyo3 0.30.1`, replaced by `Bound<'py, PyAny>::cast::<T>()` and `cast_into::<T>()`.
+- **Upstream technical rationale**: Unified smart-reference type conversion naming across the `Bound<'py, T>` API.
+
+### D. PyO3 0.29.0 - 0.30.1 & Python 3.15: Stable ABI `abi3t` (PEP 803) and `#[pyclass(frozen, subclass)]`
+- **What changed**:
+  1. Added the `abi3t` feature alongside `abi3-py310` to support the cross-version Stable ABI across Python 3.10–3.15 on both standard GIL builds and free-threaded builds (`python3.14t` / `python3.15t`).
+  2. Rust classes exposed to Python via `#[pyclass]` no longer permit Python-side subclassing by default unless the `subclass` attribute is explicitly specified, and `frozen` is recommended for lock-free concurrency without `PyCell` borrow-checking overhead in free-threaded mode.
+  3. Inside declarative `#[pymodule] mod ...` blocks, external sub-components are registered using functions annotated with `#[pymodule_init]`.
+
+---
+
+## 2.3 Adaptations in LuminaMoon Source Code
+
+1. **`Cargo.toml` & `data/Cargo.toml`**:
+   - Configured `pyo3 = { version = "0.30.1", features = ["abi3-py310", "abi3t"] }` in `[workspace.dependencies]` without `"extension-module"`, allowing `cargo test -p luminamoon-dataloader` to link and run standalone.
+2. **Runtime Detachment via `py.detach` (`data/src/py_bridge.rs`)**:
+   - All disk I/O, `mmap` reads, HTTP 206 network streaming, Parquet schema inference, and HuggingFace Hub NDJSON uploads execute inside `py.detach(|| ...)`:
+     ```rust
+     let batch_opt = py
+         .detach(|| self.inner.next_batch())
+         .map_err(to_py_err)?;
+     ```
+3. **Type Casting via `Bound::cast::<T>()` and `PyBool` vs `PyInt` Hierarchy (`data/src/py_bridge.rs`)**:
+   - Uses `columns.cast::<PyDict>()`, `v.cast::<PySequence>()`, and `pair.cast::<PyTuple>()` inside `extract_py_columns_table`.
+   - Because `bool` is a subclass of `int` in CPython, `py_any_to_multimodal_value` always checks `item.is_instance_of::<PyBool>()` before `item.is_instance_of::<PyInt>()` so boolean columns are never coerced into `1`/`0` integers.
+4. **`#[pyclass(frozen, subclass)]` Declarations and `#[pymodule_init]` Registration (`data/src/py_bridge.rs` & `optimizer/src/lib.rs`)**:
+   - `PyDataStreamLoader`, `PyMultimodalParquetLoader`, and `PyHfHubUploader` are declared with `#[pyclass(frozen, subclass, ...)]` and synchronized on the Rust side using `parking_lot::Mutex`.
+   - `register_dataloader_classes(m)` is invoked from `#[pymodule_init] fn init(m: &Bound<'_, PyModule>)` inside `optimizer/src/lib.rs`, registering all classes and module-level functions (`serialize_table_to_bytes`, `postprocess_and_dedup_records`, `suggest_closest_matches`).
+5. **Automatic Row-to-Column Transposition & GIL-Free Table/Postprocess Functions (`data/src/py_bridge.rs`)**:
+   - `extract_py_columns_table` supports three Python input layouts simultaneously: `dict[str, Sequence[Any]]`, `Sequence[tuple[str, Sequence[Any]]]`, and `Sequence[dict[str, Any]]` (transposing row dictionaries into ordered columns by first-seen key order and filling missing keys with `MultimodalValue::Null`).
+   - `serialize_table_to_bytes` and `postprocess_and_dedup_records` (with `deduplicate: bool = true`) execute Parquet/Arrow IPC/CSV/JSONL/JSON serialization and Rayon + SIMD normalization + `xxh3_64` deduplication completely outside the GIL via `py.detach(|| ...)`.

data/docs/library_audit/03_parquet_arrow_and_http206_streaming.md

@@ -1,48 +1,48 @@
-# 3. Apache Arrow/Parquet, `reqwest`, `tokio`, `bytes`, and `futures`
-
-## 3.1 Latest Stable Versions
-- **`parquet`**: `60.0.0` (`features = ["async"]`)
-- **`arrow-array`**: `60.0.0`
-- **`arrow-schema`**: `60.0.0`
-- **`reqwest`**: `0.13.5` (`default-features = false`, `features = ["default-tls", "http2", "system-proxy", "json", "stream", "gzip"]`)
-- **`tokio`**: `1.53.3` (`features = ["rt-multi-thread", "fs", "io-util", "sync", "time", "macros"]`)
-- **`bytes`**: `1.12.1`
-- **`futures`**: `0.3.34`
-
----
-
-## 3.2 Version Range Changelog (`arrow-rs 53.0.0` - `60.0.0`, `reqwest 0.12` - `0.13.5`, `bytes 1.6` - `1.12.1`)
-
-### A. Apache Arrow & Parquet (`53.0.0` - `60.0.0`)
-- **`Utf8View` (`StringViewArray`) and `BinaryView` (`BinaryViewArray`) Type Promotion**:
-  Across `arrow 54.0`–`60.0.0`, modern Parquet readers can map string and binary columns into *ByteView* representations (12-byte inline prefix + offset buffer) alongside `StringArray`/`LargeStringArray` and `BinaryArray`/`LargeBinaryArray`.
-- **Separation of `ArrowReaderMetadata` for Repeated Reads**:
-  `ParquetRecordBatchReaderBuilder::try_new(file)` re-reads and re-parses the Parquet footer metadata on every invocation. The `ArrowReaderMetadata::load(&file, options)` API paired with `ParquetRecordBatchReaderBuilder::new_with_metadata(file, metadata)` allows sharing cached metadata in memory in $O(1)$ time.
-- **`ParquetMetaDataReader::load_and_finish` Signature Change (`parquet 58.0` - `60.0.0`)**:
-  The async method `ParquetMetaDataReader::load_and_finish` now takes a generic parameter `F: MetadataSuffixFetch` (blanket-implemented for `&mut T` where `T: AsyncFileReader`). Calling `.load_and_finish(self, file_size)` directly inside `fn get_metadata(&'a mut self, ...)` fails trait-bound resolution because `self` has type `&'a mut Self`, requiring an explicit reborrow `&mut *self`.
-
-### B. `reqwest` (`0.12.0` - `0.13.5`) & `tokio` (`1.38.0` - `1.53.3`)
-- **Default TLS Backend Change in `reqwest 0.13.x`**:
-  Starting in `reqwest 0.13.0`, default features (`default-features = true`) switched from `native-tls` to `rustls` backed by `aws-lc-sys`. On Windows, compiling `aws-lc-sys` requires external CMake and NASM installations, causing build failures when absent.
-- **Modularization of `json`, `stream`, and `gzip` Features**:
-  `.json(...)`, `.bytes_stream()`, and automatic response decompression in `reqwest 0.13.5` are gated behind explicit feature flags.
-
-### C. `bytes` (`1.6.0` - `1.12.1`): Mitigation of `RUSTSEC-2026-0007` / `CVE-2026-25541`
-- **What changed**: In `bytes < 1.11.1`, an integer overflow vulnerability existed in capacity calculations inside `BytesMut::reserve` when reservation sizes approached `usize::MAX`, potentially allocating a smaller buffer than requested (*heap buffer overflow*).
-- **Upstream fix (`1.11.1` - `1.12.1`)**: Added checked arithmetic across all `BytesMut` capacity expansion paths and zero-copy `Bytes::slice` operations.
-
----
-
-## 3.3 Adaptations in LuminaMoon Source Code
-
-1. **`StringViewArray` and `BinaryViewArray` Support (`data/src/source_local.rs` & `data/src/multimodal.rs`)**:
-   - `extract_strings_from_batch` and `extract_array_values` handle all three Arrow string/binary type families (`StringArray` / `LargeStringArray` / `StringViewArray` and `BinaryArray` / `LargeBinaryArray` / `BinaryViewArray`).
-2. **`ArrowReaderMetadata` Caching and `RowSelection` (`data/src/source_local.rs` & `data/src/multimodal.rs`)**:
-   - `LocalParquetSource` and `LocalMultimodalParquetSource` load `ArrowReaderMetadata` once during `open()` and combine it with `ProjectionMask::roots(...)` and `RowSelection::from(vec![RowSelector::skip(...), RowSelector::select(...)])`.
-3. **Explicit `&mut *self` Reborrow on `HttpParquetReader` (`data/src/source_hf.rs`)**:
-   - `AsyncFileReader::get_metadata` for `HttpParquetReader` calls `ParquetMetaDataReader::new().with_arrow_reader_options(options).load_and_finish(&mut *self, file_size).await?` and caches the result in `cached_metadata: Option<Arc<ParquetMetaData>>`.
-4. **`reqwest 0.13.5` and `bytes 1.12.1` Configuration (`Cargo.toml`)**:
-   - Configured `reqwest = { version = "0.13.5", default-features = false, features = ["default-tls", "http2", "system-proxy", "json", "stream", "gzip"] }` and `bytes = "1.12.1"` so cross-platform builds remain free of external `aws-lc-sys` CMake/NASM dependencies while mitigating `CVE-2026-25541`.
-5. **Concurrent Multi-Range Downloads (`fetch_ranges_concurrent`) & Multi-Thread Tokio Pool (`data/src/source_hf.rs` & `data/src/preram.rs`)**:
-   - `HttpRangeClient::fetch_ranges_concurrent` uses `futures::stream::iter(...).buffered(concurrency)` to download multiple HTTP 206 byte ranges in parallel across workers while preserving range ordering.
-   - `HttpParquetReader::get_byte_ranges` forwards Arrow/Parquet column range lists to `fetch_ranges_concurrent`, while `RemoteLineSource` and `PreRamDataLoader` run a multi-threaded Tokio runtime (`Builder::new_multi_thread().worker_threads(download_workers)`) for parallel multi-chunk prefetching.
+# 3. Apache Arrow/Parquet, `reqwest`, `tokio`, `bytes`, and `futures`
+
+## 3.1 Latest Stable Versions
+- **`parquet`**: `60.0.0` (`features = ["async"]`)
+- **`arrow-array`**: `60.0.0`
+- **`arrow-schema`**: `60.0.0`
+- **`reqwest`**: `0.13.5` (`default-features = false`, `features = ["default-tls", "http2", "system-proxy", "json", "stream", "gzip"]`)
+- **`tokio`**: `1.53.3` (`features = ["rt-multi-thread", "fs", "io-util", "sync", "time", "macros"]`)
+- **`bytes`**: `1.12.1`
+- **`futures`**: `0.3.34`
+
+---
+
+## 3.2 Version Range Changelog (`arrow-rs 53.0.0` - `60.0.0`, `reqwest 0.12` - `0.13.5`, `bytes 1.6` - `1.12.1`)
+
+### A. Apache Arrow & Parquet (`53.0.0` - `60.0.0`)
+- **`Utf8View` (`StringViewArray`) and `BinaryView` (`BinaryViewArray`) Type Promotion**:
+  Across `arrow 54.0`–`60.0.0`, modern Parquet readers can map string and binary columns into *ByteView* representations (12-byte inline prefix + offset buffer) alongside `StringArray`/`LargeStringArray` and `BinaryArray`/`LargeBinaryArray`.
+- **Separation of `ArrowReaderMetadata` for Repeated Reads**:
+  `ParquetRecordBatchReaderBuilder::try_new(file)` re-reads and re-parses the Parquet footer metadata on every invocation. The `ArrowReaderMetadata::load(&file, options)` API paired with `ParquetRecordBatchReaderBuilder::new_with_metadata(file, metadata)` allows sharing cached metadata in memory in $O(1)$ time.
+- **`ParquetMetaDataReader::load_and_finish` Signature Change (`parquet 58.0` - `60.0.0`)**:
+  The async method `ParquetMetaDataReader::load_and_finish` now takes a generic parameter `F: MetadataSuffixFetch` (blanket-implemented for `&mut T` where `T: AsyncFileReader`). Calling `.load_and_finish(self, file_size)` directly inside `fn get_metadata(&'a mut self, ...)` fails trait-bound resolution because `self` has type `&'a mut Self`, requiring an explicit reborrow `&mut *self`.
+
+### B. `reqwest` (`0.12.0` - `0.13.5`) & `tokio` (`1.38.0` - `1.53.3`)
+- **Default TLS Backend Change in `reqwest 0.13.x`**:
+  Starting in `reqwest 0.13.0`, default features (`default-features = true`) switched from `native-tls` to `rustls` backed by `aws-lc-sys`. On Windows, compiling `aws-lc-sys` requires external CMake and NASM installations, causing build failures when absent.
+- **Modularization of `json`, `stream`, and `gzip` Features**:
+  `.json(...)`, `.bytes_stream()`, and automatic response decompression in `reqwest 0.13.5` are gated behind explicit feature flags.
+
+### C. `bytes` (`1.6.0` - `1.12.1`): Mitigation of `RUSTSEC-2026-0007` / `CVE-2026-25541`
+- **What changed**: In `bytes < 1.11.1`, an integer overflow vulnerability existed in capacity calculations inside `BytesMut::reserve` when reservation sizes approached `usize::MAX`, potentially allocating a smaller buffer than requested (*heap buffer overflow*).
+- **Upstream fix (`1.11.1` - `1.12.1`)**: Added checked arithmetic across all `BytesMut` capacity expansion paths and zero-copy `Bytes::slice` operations.
+
+---
+
+## 3.3 Adaptations in LuminaMoon Source Code
+
+1. **`StringViewArray` and `BinaryViewArray` Support (`data/src/source_local.rs` & `data/src/multimodal.rs`)**:
+   - `extract_strings_from_batch` and `extract_array_values` handle all three Arrow string/binary type families (`StringArray` / `LargeStringArray` / `StringViewArray` and `BinaryArray` / `LargeBinaryArray` / `BinaryViewArray`).
+2. **`ArrowReaderMetadata` Caching and `RowSelection` (`data/src/source_local.rs` & `data/src/multimodal.rs`)**:
+   - `LocalParquetSource` and `LocalMultimodalParquetSource` load `ArrowReaderMetadata` once during `open()` and combine it with `ProjectionMask::roots(...)` and `RowSelection::from(vec![RowSelector::skip(...), RowSelector::select(...)])`.
+3. **Explicit `&mut *self` Reborrow on `HttpParquetReader` (`data/src/source_hf.rs`)**:
+   - `AsyncFileReader::get_metadata` for `HttpParquetReader` calls `ParquetMetaDataReader::new().with_arrow_reader_options(options).load_and_finish(&mut *self, file_size).await?` and caches the result in `cached_metadata: Option<Arc<ParquetMetaData>>`.
+4. **`reqwest 0.13.5` and `bytes 1.12.1` Configuration (`Cargo.toml`)**:
+   - Configured `reqwest = { version = "0.13.5", default-features = false, features = ["default-tls", "http2", "system-proxy", "json", "stream", "gzip"] }` and `bytes = "1.12.1"` so cross-platform builds remain free of external `aws-lc-sys` CMake/NASM dependencies while mitigating `CVE-2026-25541`.
+5. **Concurrent Multi-Range Downloads (`fetch_ranges_concurrent`) & Multi-Thread Tokio Pool (`data/src/source_hf.rs` & `data/src/preram.rs`)**:
+   - `HttpRangeClient::fetch_ranges_concurrent` uses `futures::stream::iter(...).buffered(concurrency)` to download multiple HTTP 206 byte ranges in parallel across workers while preserving range ordering.
+   - `HttpParquetReader::get_byte_ranges` forwards Arrow/Parquet column range lists to `fetch_ranges_concurrent`, while `RemoteLineSource` and `PreRamDataLoader` run a multi-threaded Tokio runtime (`Builder::new_multi_thread().worker_threads(download_workers)`) for parallel multi-chunk prefetching.

data/docs/library_audit/04_xxh3_dedup_crc32_and_hf_hub_security.md

@@ -1,49 +1,49 @@
-# 4. `xxhash-rust`, `hashbrown`, `crc32fast`, `parking_lot`, and HuggingFace Hub API Specifications
-
-## 4.1 Latest Stable Versions
-- **`xxhash-rust`**: `0.8.19` (`features = ["xxh3"]`)
-- **`hashbrown`**: `0.17.1`
-- **`crc32fast`**: `1.5.2`
-- **`parking_lot`**: `0.12.5`
-- **External HuggingFace Hub Protocol**: `huggingface_hub >= 0.28 / 1.x` (HTTP Commit NDJSON API & Dataset Viewer Parquet Spec)
-
----
-
-## 4.2 Version Range Changelog (`2024` - `2026`)
-
-### A. `hashbrown` (`0.14.0` - `0.17.1`)
-- **What changed**: Starting in `hashbrown 0.15.0` through `0.17.1`, the internal default hasher for `HashMap` and `HashSet` switched from `ahash` to `foldhash`.
-- **SwissTable technical implications**: `hashbrown` uses the SwissTable architecture, splitting a 64-bit hash into two parts: the lower 57 bits (`h1`) for bucket slot selection and the upper 7 bits (`h2`, bits 57..63) as the control byte for SIMD group probing. When table keys are already uniformly distributed 64-bit hashes (`u64`), running `foldhash` on top of them performs redundant computation.
-
-### B. `xxhash-rust` (`0.8.10` - `0.8.19`)
-- **What changed**: Enhanced `const fn` compilation optimizations and auto-vectorization in `xxh3_64(input: &[u8]) -> u64`.
-- **Bit distribution properties**: `xxh3_64` provides full avalanche across all 64 bits (including the upper 7 `h2` bits), making it safe to pair directly with SwissTable via an identity hasher without re-hashing.
-
-### C. `crc32fast` (`1.4.0` - `1.5.2`)
-- **What changed**: The `1.5.x` series enables `vpclmulqdq` hardware instructions (Vector Carry-Less Multiplication on AVX-512) alongside classic `pclmulqdq` to accelerate IEEE CRC32 checksum calculation.
-
-### D. `parking_lot` (`0.12.1` - `0.12.5`)
-- **What changed**: Maintenance updates for Rust Edition 2024 lint compatibility across `Mutex<T>` and `Condvar` synchronization primitives with zero OS mutex allocation on the uncontended fast path.
-
-### E. HuggingFace Hub Credential & External Endpoint Conventions (`huggingface_hub 0.25` - `1.x`)
-- **Standard environment variables**: The official `huggingface_hub` library standardized token resolution priority on `HF_TOKEN` (superseding legacy `HUGGING_FACE_HUB_TOKEN` as primary), local token files at `$HF_HOME/token` or `~/.cache/huggingface/token`, and `HF_ENDPOINT` (defaulting to `https://huggingface.co`).
-- **`application/x-ndjson` Commit Protocol**: The `POST /api/datasets/{owner}/{repo}/commit/{revision}` endpoint accepts Newline-Delimited JSON streams beginning with `{"key": "header", "value": {"summary": ...}}` followed by `{"key": "file", "value": {"content": "<base64>", "path": ..., "encoding": "base64"}}`.
-- **Dataset Viewer Image Schema Convention**: For image columns in Parquet files to render automatically in the HuggingFace Dataset Viewer without manual configuration, image columns must be stored as an Arrow `Struct` with two fields: `bytes: Binary` and `path: Utf8`.
-
----
-
-## 4.3 Adaptations in LuminaMoon Source Code
-
-1. **Bypassing `foldhash` with `IdentityBuildHasher` + `xxh3_64` (`data/src/dedup.rs`)**:
-   - Defines `IdentityHasher` (`write_u64` stores the `xxh3_64` hash directly) for `HashSet<u64, IdentityBuildHasher>` and `HashMap<u64, u32, IdentityBuildHasher>`, eliminating `foldhash` double-hashing overhead in `hashbrown 0.17.1`.
-2. **Checkpoint Slot Validation via `crc32fast::Hasher` (`data/src/metadata.rs`)**:
-   - Uses `crc32fast` to verify the integrity of both 64-byte Ping-Pong checkpoint slots (`Slot A` and `Slot B`).
-3. **Fine-Grained `parking_lot::Mutex` Separation (`data/src/preram.rs`)**:
-   - Separates `Mutex<QueueState>` and `Condvar` from `Mutex<DualMetadataManager>` so background Pre-RAM refill workers and `commit_gpu_batch()` callers never block in-memory batch pops.
-4. **HuggingFace Hub Protocol Compliance (`data/src/hf_hub.rs` & `data/src/multimodal_serialize.rs`)**:
-   - `HfCredentials` resolves `HF_TOKEN`, `HUGGING_FACE_HUB_TOKEN`, local token files, `HF_ORG`, `HF_USERNAME`, and `HF_ENDPOINT`, redacts token values as `"[REDACTED]"` in `fmt::Debug`, and blocks token transmission over non-loopback `http://` connections.
-   - `serialize_columns_to_parquet_bytes_with_options` encodes `MultimodalValue::Bytes` columns into `{bytes: Binary, path: Utf8}` `StructArray` layouts directly compatible with the HuggingFace Dataset Viewer.
-5. **Colab/Kaggle Credential Auto-Resolver, RFC 5988 `Link` Pagination, and Hybrid LFS/Xet Upload (`python/luminamoon/data/hub.py`)**:
-   - `resolve_hf_secrets` automatically resolves `HF_TOKEN`, `HF_ORG`, and `HF_USERNAME` from environment variables, `google.colab.userdata`, or `kaggle_secrets.UserSecretsClient`.
-   - `expand_hf_sources` supports automatic shard discovery from repository roots `hf://datasets/org/repo[@revision]` via `huggingface_hub.HfApi.list_repo_files` or the REST `/api/datasets/.../tree/...?recursive=true` endpoint with RFC 5988 `Link: <...>; rel="next"` header pagination and per-page HTTPS validation.
-   - `push_to_hub` combines GIL-free Rust serialization (`serialize_table_to_bytes`) with `huggingface_hub.HfApi.upload_file` (supporting Git LFS / Xet multipart uploads for large shards) and automatic fallback to the native Rust `HfHubUploader` (`application/x-ndjson`).
+# 4. `xxhash-rust`, `hashbrown`, `crc32fast`, `parking_lot`, and HuggingFace Hub API Specifications
+
+## 4.1 Latest Stable Versions
+- **`xxhash-rust`**: `0.8.19` (`features = ["xxh3"]`)
+- **`hashbrown`**: `0.17.1`
+- **`crc32fast`**: `1.5.2`
+- **`parking_lot`**: `0.12.5`
+- **External HuggingFace Hub Protocol**: `huggingface_hub >= 0.28 / 1.x` (HTTP Commit NDJSON API & Dataset Viewer Parquet Spec)
+
+---
+
+## 4.2 Version Range Changelog (`2024` - `2026`)
+
+### A. `hashbrown` (`0.14.0` - `0.17.1`)
+- **What changed**: Starting in `hashbrown 0.15.0` through `0.17.1`, the internal default hasher for `HashMap` and `HashSet` switched from `ahash` to `foldhash`.
+- **SwissTable technical implications**: `hashbrown` uses the SwissTable architecture, splitting a 64-bit hash into two parts: the lower 57 bits (`h1`) for bucket slot selection and the upper 7 bits (`h2`, bits 57..63) as the control byte for SIMD group probing. When table keys are already uniformly distributed 64-bit hashes (`u64`), running `foldhash` on top of them performs redundant computation.
+
+### B. `xxhash-rust` (`0.8.10` - `0.8.19`)
+- **What changed**: Enhanced `const fn` compilation optimizations and auto-vectorization in `xxh3_64(input: &[u8]) -> u64`.
+- **Bit distribution properties**: `xxh3_64` provides full avalanche across all 64 bits (including the upper 7 `h2` bits), making it safe to pair directly with SwissTable via an identity hasher without re-hashing.
+
+### C. `crc32fast` (`1.4.0` - `1.5.2`)
+- **What changed**: The `1.5.x` series enables `vpclmulqdq` hardware instructions (Vector Carry-Less Multiplication on AVX-512) alongside classic `pclmulqdq` to accelerate IEEE CRC32 checksum calculation.
+
+### D. `parking_lot` (`0.12.1` - `0.12.5`)
+- **What changed**: Maintenance updates for Rust Edition 2024 lint compatibility across `Mutex<T>` and `Condvar` synchronization primitives with zero OS mutex allocation on the uncontended fast path.
+
+### E. HuggingFace Hub Credential & External Endpoint Conventions (`huggingface_hub 0.25` - `1.x`)
+- **Standard environment variables**: The official `huggingface_hub` library standardized token resolution priority on `HF_TOKEN` (superseding legacy `HUGGING_FACE_HUB_TOKEN` as primary), local token files at `$HF_HOME/token` or `~/.cache/huggingface/token`, and `HF_ENDPOINT` (defaulting to `https://huggingface.co`).
+- **`application/x-ndjson` Commit Protocol**: The `POST /api/datasets/{owner}/{repo}/commit/{revision}` endpoint accepts Newline-Delimited JSON streams beginning with `{"key": "header", "value": {"summary": ...}}` followed by `{"key": "file", "value": {"content": "<base64>", "path": ..., "encoding": "base64"}}`.
+- **Dataset Viewer Image Schema Convention**: For image columns in Parquet files to render automatically in the HuggingFace Dataset Viewer without manual configuration, image columns must be stored as an Arrow `Struct` with two fields: `bytes: Binary` and `path: Utf8`.
+
+---
+
+## 4.3 Adaptations in LuminaMoon Source Code
+
+1. **Bypassing `foldhash` with `IdentityBuildHasher` + `xxh3_64` (`data/src/dedup.rs`)**:
+   - Defines `IdentityHasher` (`write_u64` stores the `xxh3_64` hash directly) for `HashSet<u64, IdentityBuildHasher>` and `HashMap<u64, u32, IdentityBuildHasher>`, eliminating `foldhash` double-hashing overhead in `hashbrown 0.17.1`.
+2. **Checkpoint Slot Validation via `crc32fast::Hasher` (`data/src/metadata.rs`)**:
+   - Uses `crc32fast` to verify the integrity of both 64-byte Ping-Pong checkpoint slots (`Slot A` and `Slot B`).
+3. **Fine-Grained `parking_lot::Mutex` Separation (`data/src/preram.rs`)**:
+   - Separates `Mutex<QueueState>` and `Condvar` from `Mutex<DualMetadataManager>` so background Pre-RAM refill workers and `commit_gpu_batch()` callers never block in-memory batch pops.
+4. **HuggingFace Hub Protocol Compliance (`data/src/hf_hub.rs` & `data/src/multimodal_serialize.rs`)**:
+   - `HfCredentials` resolves `HF_TOKEN`, `HUGGING_FACE_HUB_TOKEN`, local token files, `HF_ORG`, `HF_USERNAME`, and `HF_ENDPOINT`, redacts token values as `"[REDACTED]"` in `fmt::Debug`, and blocks token transmission over non-loopback `http://` connections.
+   - `serialize_columns_to_parquet_bytes_with_options` encodes `MultimodalValue::Bytes` columns into `{bytes: Binary, path: Utf8}` `StructArray` layouts directly compatible with the HuggingFace Dataset Viewer.
+5. **Colab/Kaggle Credential Auto-Resolver, RFC 5988 `Link` Pagination, and Hybrid LFS/Xet Upload (`python/luminamoon/data/hub.py`)**:
+   - `resolve_hf_secrets` automatically resolves `HF_TOKEN`, `HF_ORG`, and `HF_USERNAME` from environment variables, `google.colab.userdata`, or `kaggle_secrets.UserSecretsClient`.
+   - `expand_hf_sources` supports automatic shard discovery from repository roots `hf://datasets/org/repo[@revision]` via `huggingface_hub.HfApi.list_repo_files` or the REST `/api/datasets/.../tree/...?recursive=true` endpoint with RFC 5988 `Link: <...>; rel="next"` header pagination and per-page HTTPS validation.
+   - `push_to_hub` combines GIL-free Rust serialization (`serialize_table_to_bytes`) with `huggingface_hub.HfApi.upload_file` (supporting Git LFS / Xet multipart uploads for large shards) and automatic fallback to the native Rust `HfHubUploader` (`application/x-ndjson`).

data/docs/library_audit/05_universal_formats_compression_and_diagnostics.md

@@ -1,87 +1,87 @@
-# 5. Universal Formats (`arrow-ipc`, `half`), Stream Decompression (`flate2`, `zstd`, `lz4_flex`), and Typo Diagnostics
-
-## 5.1 Latest Stable Versions (October 2026)
-- **`arrow-ipc`**: `60.0.0` (synchronized with `arrow-array = "60.0.0"`, `arrow-schema = "60.0.0"`, `parquet = "60.0.0"`)
-- **`half`**: `2.7.1` (`f16` / `bf16` IEEE 754-2008 half-precision floating point)
-- **`flate2`**: `1.1.10` (`features = ["rust_backend"]` / `miniz_oxide 0.8.9` + `crc32fast 1.5.2`)
-- **`zstd`**: `0.14.3` (`zstd-safe 8.2.0` + `zstd-sys 2.0.16+zstd.1.5.7`)
-- **`lz4_flex`**: `0.14.0` (`safe-decode` & `frame` enabled by default, `unsafe`-free decompression path)
-
----
-
-## 5.2 Detailed Version-by-Version Changelog over a 1-Year Window (`2025` - `October 2026`)
-
-### A. `arrow-ipc` (`53.0.0` - `60.0.0`)
-| Version | Date / Era | API, ABI, and Internal Behavior Changes |
-| :--- | :--- | :--- |
-| **`53.0.0` - `54.0.0`** | Q4 2024 - Q1 2025 | Added full IPC serialization and deserialization support (`FileReader`, `StreamReader`, `FileWriter`, `StreamWriter`) for variable-length view types `Utf8View` (`StringViewArray`) and `BinaryView` (`BinaryViewArray`) with dedicated variadic buffers. |
-| **`55.0.0` - `56.0.0`** | Q2 2025 | Strict 8-byte / 64-byte buffer alignment validation on IPC `RecordBatch` messages to prevent unaligned pointer reads when memory-mapping `.arrow` / `.feather` files (`mmap` or `Cursor<&[u8]>`). |
-| **`57.0.0`** | Q3 2025 | Improved IPC body compression (`LZ4_FRAME` and `ZSTD`) and variadic buffer bounds checking on `BinaryView` to prevent out-of-bounds slice panics on corrupted IPC streams. |
-| **`58.0.0`** | Q4 2025 | Deprecated `StreamReader::try_new_unbuffered`; unified internal buffering in `StreamReader::try_new` and raised MSRV to match modern Rust toolchains. |
-| **`59.0.0` - `60.0.0`** | 2026 | Removed legacy methods deprecated in `57.x`/`58.x`, aligned `Field` and `Schema` structures with `arrow-schema 60.0.0`, and enabled zero-copy slicing across `ListArray`, `LargeListArray`, `FixedSizeListArray`, and `Float16Array`. |
-
-### B. `lz4_flex` (`0.11.0` - `0.14.0`) & Mitigation of `CVE-2026-32829` (`RUSTSEC-2026-0041`)
-| Version | Release Date | API Changes, Security Features, and CVE Mitigations |
-| :--- | :--- | :--- |
-| **`0.11.0` - `0.11.5`** | 2024 - 2025 | Inverted the feature flag from `checked-decode` to `unchecked-decode`. Previously, additive Cargo feature unification meant a transitive dependency disabling `checked-decode` could silently enable `unsafe` pointer decoding. Inverting the flag to `unchecked-decode` (off by default) with `safe-decode` (on by default) maintains `unsafe`-free decompression unless explicitly opted in. |
-| **`0.11.6` & `0.12.0` - `0.12.1`** | **2026-05-09** | **Critical Security Mitigation `CVE-2026-32829` / `RUSTSEC-2026-0041` (CVSS 7.5 High)**: In `lz4_flex < 0.11.6` and `0.12.0`, when using APIs that write into uninitialized output buffers (`decompress_into` / `Sink`) on malformed LZ4 inputs containing an invalid match offset beyond already-written bytes, the decompressor could copy uninitialized memory bytes into the output buffer before returning an error (*information leak*). Fixed in `0.11.6` and `0.12.1` by validating match offset bounds against the actual output cursor prior to copying. |
-| **`0.13.0`** | **2026-05-09** | Added external dictionary compression/decompression support (`compress_prepend_size_with_dict`, `decompress_size_prepended_with_dict`) for small LZ4 blocks. |
-| **`0.13.1`** | **2026-05-09** | Fixed an out-of-bounds panic when an external dictionary smaller than 4 bytes (`dict.len() < 4`) was passed to dictionary compression functions. |
-| **`0.14.0`** | **2026-07-14** | Explicitly separated the `alloc` feature flag from `no_std` paths, refined `frame::FrameDecoder` (`std::io::Read`) for framed `.lz4` streams (*LZ4 Frame Format*), and optimized `safe-decode` throughput. |
-
-### C. `flate2` (`1.0.30` - `1.1.10`)
-| Version | Release Date | Backend, Performance, and Feature Flag Changes |
-| :--- | :--- | :--- |
-| **`1.0.30` - `1.0.35`** | 2024 - Early 2025 | Upgraded `miniz_oxide` to the `0.8.x` series and introduced optional `zlib-rs` backend support (pure-Rust SIMD-accelerated Zlib). |
-| **`1.1.0` - `1.1.4`** | Mid 2025 | Added `.reset()` and internal buffer reuse across decoders (`GzDecoder`, `ZlibDecoder`, `DeflateDecoder`), eliminating repeated heap allocations when decompressing thousands of small `.gz` shards. |
-| **`1.1.5` - `1.1.8`** | Late 2025 - Early 2026 | Optimized `MultiGzDecoder` for concatenated `.jsonl.gz` streams and improved handling of trailing bytes after Gzip CRC32 footers. |
-| **`1.1.9`** | Q2 2026 | Refactored internal `zlib-rs` and `crc32fast` feature flags, which temporarily disabled runtime CPU SIMD detection (`std`) when consumers customized default features. |
-| **`1.1.10`** | **Q3 2026** | **Throughput Regression Fix (`runtime_detection`)**: Added the `runtime_detection` feature flag (`zlib-rs?/std`, `crc32fast?/std`) directly into `default` features, ensuring `crc32fast` (`vpclmulqdq` / `pclmulqdq` / `asimd`) and compression backends automatically detect CPU SIMD capabilities at runtime. |
-
-### D. `zstd` (`0.13.0` - `0.14.3`) & `zstd-safe` (`7.2.0` - `8.2.0`)
-| Version | Release Date | Memory Safety (`Soundness`) and C `libzstd` Engine Changes |
-| :--- | :--- | :--- |
-| **`0.13.0` - `0.13.3`** | 2024 - 2025 | Upgraded `zstd-sys` bindings to `libzstd 1.5.6` and `1.5.7` (faster large-frame decompression and rare long-distance matcher bug fixes). |
-| **`0.14.0`** | **2026** | **`zstd-safe 8.0.0+` Soundness Fix (`CCtx` / `DCtx`)**: Resolved a memory-safety soundness bug where streaming wrappers allowed direct `&mut CCtx` / `&mut DCtx` borrows (`DerefMut`), permitting callers to `std::mem::swap` contexts while holding raw pointers to short-lived dictionary buffers (*lifetime violation*). In `0.14.x`, direct mutable context access is restricted in favor of safe delegation methods on `Encoder` and `Decoder`. |
-| **`0.14.1` - `0.14.3`** | **2026** | Updated dependencies to `zstd-safe ^8.2.0` (`zstd 1.5.7`), refined `zstd::stream::read::Decoder` and `zstd::stream::encode_all` for multi-frame `.zst` streams (including HuggingFace Xet/Parquet skippable metadata frames), and eliminated compiler warnings on Rust `1.99.1`. |
-
-### E. `half` (`2.4.0` - `2.7.1`)
-| Version | Release Date | `f16` / `bf16` Conversion and Hardware Intrinsic Changes |
-| :--- | :--- | :--- |
-| **`2.4.0` - `2.5.0`** | 2024 - 2025 | Stabilized batch slice conversions (`f16::to_f32_slice`, `bf16::to_f32_slice`) and full `bytemuck` and `serde` integration. |
-| **`2.6.0`** | 2025 | Raised MSRV to Rust `1.81+`, utilizing `f16c` / `avx512fp16` intrinsics on `x86_64` and `fp16` on `aarch64` on stable compilers without nightly feature flags. |
-| **`2.7.0` - `2.7.1`** | **2026** | Full alignment with `arrow-array 60.0.0` (`Float16Array` exports `half::f16` elements), branchless `to_f64()` optimization for subnormal and `NaN` values, and full compliance with Rust Edition 2024 `unsafe_op_in_unsafe_fn` rules. |
-
-### F. Self-Contained WebDataset `.tar` Parser & Typo Diagnostics (Self-Contained Implementation)
-- **Why External `tar` Crates Were Avoided (`CVE-2025-62518` / TARmageddon)**:
-  In late 2025, a critical vulnerability (**`CVE-2025-62518` / TARmageddon**) was disclosed across async TAR parsers in the Rust ecosystem (`tokio-tar` / `astral-tokio-tar`) caused by desynchronization between PAX extended headers and GNU longname headers, enabling file smuggling and path traversal.
-- **LuminaMoon's Self-Contained Solution (`parse_webdataset_tar_bytes` in `data/src/multimodal_formats.rs`)**:
-  Because WebDataset ingestion only groups paired in-memory payloads (e.g., `sample_001.jpg`, `sample_001.txt`, `sample_001.json`) into tabular rows without ever extracting files to disk, LuminaMoon implements a pure in-memory 512-byte USTAR block parser that strictly rejects directory/symlink entries and deterministically groups sample stems.
-
----
-
-## 5.3 Adaptations in LuminaMoon Source Code
-
-1. **Universal Format Support (`data/src/multimodal_formats.rs`, `data/src/multimodal_serialize.rs`, `data/src/preram.rs`, `data/src/hf_hub.rs`)**:
-   - `InMemoryMultimodalTable::from_bytes_auto` automatically detects and parses:
-     * **Apache Arrow IPC / Feather v2** (`.arrow`, `.feather`, `.ipc`, or `ARROW1` magic bytes) via `arrow_ipc::reader::{FileReader, StreamReader}`.
-     * **WebDataset TAR** (`.tar`, `.tgz`, `.tar.gz`, `.tar.zst`, `.tar.lz4`), grouping sample files by stem (`00001.png` + `00001.txt` + `00001.json` $\rightarrow$ 1 multimodal row).
-     * **CSV & TSV** (`.csv`, `.tsv`) with an RFC 4180 state-machine parser (escaped quotes, automatic `Int`/`Float`/`Bool`/`Json`/`Text` type inference, and `min_score` filtering).
-     * **JSON Array / Object Wrapper & JSONL** (`.json`, `.jsonl`, `.ndjson`).
-   - `serialize_columns_by_path_format` (`data/src/hf_hub.rs`) serializes tables into `.parquet`, `.arrow`/`.feather`/`.ipc`, `.jsonl`/`.ndjson`, `.json`, `.csv`, and `.tsv`, transparently wrapping output in `.gz` (`flate2::write::GzEncoder`) or `.zst` (`zstd::stream::encode_all`) compression based on `path_in_repo`.
-
-2. **Transparent Local & Remote Stream Decompression (`data/src/source_local.rs` & `data/src/source_hf.rs`)**:
-   - `decompress_bytes_if_compressed` detects compression via both file extensions (`.gz`, `.zst`/`.zstd`, `.lz4`) and header magic bytes:
-     * **Gzip (`[0x1F, 0x8B]`)**: `flate2::read::MultiGzDecoder` (`1.1.10`).
-     * **Zstandard (`[0x28, 0xB5, 0x2F, 0xFD]`)**: `zstd::stream::decode_all` (`0.14.3`).
-     * **LZ4 Frame (`[0x04, 0x22, 0x4D, 0x18]`)**: `lz4_flex::frame::FrameDecoder` (`0.14.0`, protected against `CVE-2026-32829`).
-
-3. **Full Arrow Type Extraction & Nested JSON Paths (`data/src/multimodal.rs` & `data/src/postprocess.rs`)**:
-   - `MultimodalValue::List(Vec<MultimodalValue>)` recursively extracts `ListArray`, `LargeListArray`, and `FixedSizeListArray` (supporting multi-turn chat `messages` and embedding vectors).
-   - `Float16Array` values are converted losslessly via `half::f16::to_f64()`.
-   - `extract_json_value_by_path` supports nested dot-paths and array indices (e.g., `json_extract_key="messages.0.content"` or `"meta.quality.score"`).
-
-4. **Typo Diagnostic Engine & Developer-Friendly Errors (`data/src/error.rs` & `python/luminamoon/data/diagnostics.py`)**:
-   - **Damerau-Levenshtein Optimal String Alignment (`damerau_levenshtein`, `find_closest_matches`)**: Computes edit distance including adjacent character transpositions (e.g., `"txet"` $\rightarrow$ `"text"`, `"prmopt"` $\rightarrow$ `"prompt"`).
-   - **Fake File / Git LFS Pointer Detection (`inspect_magic_bytes_hint`)**: Detects when a `.parquet`/`.arrow` file that fails to open is actually an unfetched Git LFS text pointer (`version https://git-lfs.github.com/spec/v1`), an HTML error page (`<!DOCTYPE html>`), or a misnamed JSON/Gzip/Zstd file.
-   - **Sibling Filename & URI Suggestions (`open_local_file_with_diagnostics`, `diagnose_uri_typo`)**: Suggests similar filenames in the same directory on `FileNotFound`, corrects URI scheme typos (`hf://dataset/...` $\rightarrow$ `hf://datasets/...`, `httpss://`), and validates unknown Python `**kwargs` with closest-parameter suggestions.
+# 5. Universal Formats (`arrow-ipc`, `half`), Stream Decompression (`flate2`, `zstd`, `lz4_flex`), and Typo Diagnostics
+
+## 5.1 Latest Stable Versions (October 2026)
+- **`arrow-ipc`**: `60.0.0` (synchronized with `arrow-array = "60.0.0"`, `arrow-schema = "60.0.0"`, `parquet = "60.0.0"`)
+- **`half`**: `2.7.1` (`f16` / `bf16` IEEE 754-2008 half-precision floating point)
+- **`flate2`**: `1.1.10` (`features = ["rust_backend"]` / `miniz_oxide 0.8.9` + `crc32fast 1.5.2`)
+- **`zstd`**: `0.14.3` (`zstd-safe 8.2.0` + `zstd-sys 2.0.16+zstd.1.5.7`)
+- **`lz4_flex`**: `0.14.0` (`safe-decode` & `frame` enabled by default, `unsafe`-free decompression path)
+
+---
+
+## 5.2 Detailed Version-by-Version Changelog over a 1-Year Window (`2025` - `October 2026`)
+
+### A. `arrow-ipc` (`53.0.0` - `60.0.0`)
+| Version | Date / Era | API, ABI, and Internal Behavior Changes |
+| :--- | :--- | :--- |
+| **`53.0.0` - `54.0.0`** | Q4 2024 - Q1 2025 | Added full IPC serialization and deserialization support (`FileReader`, `StreamReader`, `FileWriter`, `StreamWriter`) for variable-length view types `Utf8View` (`StringViewArray`) and `BinaryView` (`BinaryViewArray`) with dedicated variadic buffers. |
+| **`55.0.0` - `56.0.0`** | Q2 2025 | Strict 8-byte / 64-byte buffer alignment validation on IPC `RecordBatch` messages to prevent unaligned pointer reads when memory-mapping `.arrow` / `.feather` files (`mmap` or `Cursor<&[u8]>`). |
+| **`57.0.0`** | Q3 2025 | Improved IPC body compression (`LZ4_FRAME` and `ZSTD`) and variadic buffer bounds checking on `BinaryView` to prevent out-of-bounds slice panics on corrupted IPC streams. |
+| **`58.0.0`** | Q4 2025 | Deprecated `StreamReader::try_new_unbuffered`; unified internal buffering in `StreamReader::try_new` and raised MSRV to match modern Rust toolchains. |
+| **`59.0.0` - `60.0.0`** | 2026 | Removed legacy methods deprecated in `57.x`/`58.x`, aligned `Field` and `Schema` structures with `arrow-schema 60.0.0`, and enabled zero-copy slicing across `ListArray`, `LargeListArray`, `FixedSizeListArray`, and `Float16Array`. |
+
+### B. `lz4_flex` (`0.11.0` - `0.14.0`) & Mitigation of `CVE-2026-32829` (`RUSTSEC-2026-0041`)
+| Version | Release Date | API Changes, Security Features, and CVE Mitigations |
+| :--- | :--- | :--- |
+| **`0.11.0` - `0.11.5`** | 2024 - 2025 | Inverted the feature flag from `checked-decode` to `unchecked-decode`. Previously, additive Cargo feature unification meant a transitive dependency disabling `checked-decode` could silently enable `unsafe` pointer decoding. Inverting the flag to `unchecked-decode` (off by default) with `safe-decode` (on by default) maintains `unsafe`-free decompression unless explicitly opted in. |
+| **`0.11.6` & `0.12.0` - `0.12.1`** | **2026-05-09** | **Critical Security Mitigation `CVE-2026-32829` / `RUSTSEC-2026-0041` (CVSS 7.5 High)**: In `lz4_flex < 0.11.6` and `0.12.0`, when using APIs that write into uninitialized output buffers (`decompress_into` / `Sink`) on malformed LZ4 inputs containing an invalid match offset beyond already-written bytes, the decompressor could copy uninitialized memory bytes into the output buffer before returning an error (*information leak*). Fixed in `0.11.6` and `0.12.1` by validating match offset bounds against the actual output cursor prior to copying. |
+| **`0.13.0`** | **2026-05-09** | Added external dictionary compression/decompression support (`compress_prepend_size_with_dict`, `decompress_size_prepended_with_dict`) for small LZ4 blocks. |
+| **`0.13.1`** | **2026-05-09** | Fixed an out-of-bounds panic when an external dictionary smaller than 4 bytes (`dict.len() < 4`) was passed to dictionary compression functions. |
+| **`0.14.0`** | **2026-07-14** | Explicitly separated the `alloc` feature flag from `no_std` paths, refined `frame::FrameDecoder` (`std::io::Read`) for framed `.lz4` streams (*LZ4 Frame Format*), and optimized `safe-decode` throughput. |
+
+### C. `flate2` (`1.0.30` - `1.1.10`)
+| Version | Release Date | Backend, Performance, and Feature Flag Changes |
+| :--- | :--- | :--- |
+| **`1.0.30` - `1.0.35`** | 2024 - Early 2025 | Upgraded `miniz_oxide` to the `0.8.x` series and introduced optional `zlib-rs` backend support (pure-Rust SIMD-accelerated Zlib). |
+| **`1.1.0` - `1.1.4`** | Mid 2025 | Added `.reset()` and internal buffer reuse across decoders (`GzDecoder`, `ZlibDecoder`, `DeflateDecoder`), eliminating repeated heap allocations when decompressing thousands of small `.gz` shards. |
+| **`1.1.5` - `1.1.8`** | Late 2025 - Early 2026 | Optimized `MultiGzDecoder` for concatenated `.jsonl.gz` streams and improved handling of trailing bytes after Gzip CRC32 footers. |
+| **`1.1.9`** | Q2 2026 | Refactored internal `zlib-rs` and `crc32fast` feature flags, which temporarily disabled runtime CPU SIMD detection (`std`) when consumers customized default features. |
+| **`1.1.10`** | **Q3 2026** | **Throughput Regression Fix (`runtime_detection`)**: Added the `runtime_detection` feature flag (`zlib-rs?/std`, `crc32fast?/std`) directly into `default` features, ensuring `crc32fast` (`vpclmulqdq` / `pclmulqdq` / `asimd`) and compression backends automatically detect CPU SIMD capabilities at runtime. |
+
+### D. `zstd` (`0.13.0` - `0.14.3`) & `zstd-safe` (`7.2.0` - `8.2.0`)
+| Version | Release Date | Memory Safety (`Soundness`) and C `libzstd` Engine Changes |
+| :--- | :--- | :--- |
+| **`0.13.0` - `0.13.3`** | 2024 - 2025 | Upgraded `zstd-sys` bindings to `libzstd 1.5.6` and `1.5.7` (faster large-frame decompression and rare long-distance matcher bug fixes). |
+| **`0.14.0`** | **2026** | **`zstd-safe 8.0.0+` Soundness Fix (`CCtx` / `DCtx`)**: Resolved a memory-safety soundness bug where streaming wrappers allowed direct `&mut CCtx` / `&mut DCtx` borrows (`DerefMut`), permitting callers to `std::mem::swap` contexts while holding raw pointers to short-lived dictionary buffers (*lifetime violation*). In `0.14.x`, direct mutable context access is restricted in favor of safe delegation methods on `Encoder` and `Decoder`. |
+| **`0.14.1` - `0.14.3`** | **2026** | Updated dependencies to `zstd-safe ^8.2.0` (`zstd 1.5.7`), refined `zstd::stream::read::Decoder` and `zstd::stream::encode_all` for multi-frame `.zst` streams (including HuggingFace Xet/Parquet skippable metadata frames), and eliminated compiler warnings on Rust `1.99.1`. |
+
+### E. `half` (`2.4.0` - `2.7.1`)
+| Version | Release Date | `f16` / `bf16` Conversion and Hardware Intrinsic Changes |
+| :--- | :--- | :--- |
+| **`2.4.0` - `2.5.0`** | 2024 - 2025 | Stabilized batch slice conversions (`f16::to_f32_slice`, `bf16::to_f32_slice`) and full `bytemuck` and `serde` integration. |
+| **`2.6.0`** | 2025 | Raised MSRV to Rust `1.81+`, utilizing `f16c` / `avx512fp16` intrinsics on `x86_64` and `fp16` on `aarch64` on stable compilers without nightly feature flags. |
+| **`2.7.0` - `2.7.1`** | **2026** | Full alignment with `arrow-array 60.0.0` (`Float16Array` exports `half::f16` elements), branchless `to_f64()` optimization for subnormal and `NaN` values, and full compliance with Rust Edition 2024 `unsafe_op_in_unsafe_fn` rules. |
+
+### F. Self-Contained WebDataset `.tar` Parser & Typo Diagnostics (Self-Contained Implementation)
+- **Why External `tar` Crates Were Avoided (`CVE-2025-62518` / TARmageddon)**:
+  In late 2025, a critical vulnerability (**`CVE-2025-62518` / TARmageddon**) was disclosed across async TAR parsers in the Rust ecosystem (`tokio-tar` / `astral-tokio-tar`) caused by desynchronization between PAX extended headers and GNU longname headers, enabling file smuggling and path traversal.
+- **LuminaMoon's Self-Contained Solution (`parse_webdataset_tar_bytes` in `data/src/multimodal_formats.rs`)**:
+  Because WebDataset ingestion only groups paired in-memory payloads (e.g., `sample_001.jpg`, `sample_001.txt`, `sample_001.json`) into tabular rows without ever extracting files to disk, LuminaMoon implements a pure in-memory 512-byte USTAR block parser that strictly rejects directory/symlink entries and deterministically groups sample stems.
+
+---
+
+## 5.3 Adaptations in LuminaMoon Source Code
+
+1. **Universal Format Support (`data/src/multimodal_formats.rs`, `data/src/multimodal_serialize.rs`, `data/src/preram.rs`, `data/src/hf_hub.rs`)**:
+   - `InMemoryMultimodalTable::from_bytes_auto` automatically detects and parses:
+     * **Apache Arrow IPC / Feather v2** (`.arrow`, `.feather`, `.ipc`, or `ARROW1` magic bytes) via `arrow_ipc::reader::{FileReader, StreamReader}`.
+     * **WebDataset TAR** (`.tar`, `.tgz`, `.tar.gz`, `.tar.zst`, `.tar.lz4`), grouping sample files by stem (`00001.png` + `00001.txt` + `00001.json` $\rightarrow$ 1 multimodal row).
+     * **CSV & TSV** (`.csv`, `.tsv`) with an RFC 4180 state-machine parser (escaped quotes, automatic `Int`/`Float`/`Bool`/`Json`/`Text` type inference, and `min_score` filtering).
+     * **JSON Array / Object Wrapper & JSONL** (`.json`, `.jsonl`, `.ndjson`).
+   - `serialize_columns_by_path_format` (`data/src/hf_hub.rs`) serializes tables into `.parquet`, `.arrow`/`.feather`/`.ipc`, `.jsonl`/`.ndjson`, `.json`, `.csv`, and `.tsv`, transparently wrapping output in `.gz` (`flate2::write::GzEncoder`) or `.zst` (`zstd::stream::encode_all`) compression based on `path_in_repo`.
+
+2. **Transparent Local & Remote Stream Decompression (`data/src/source_local.rs` & `data/src/source_hf.rs`)**:
+   - `decompress_bytes_if_compressed` detects compression via both file extensions (`.gz`, `.zst`/`.zstd`, `.lz4`) and header magic bytes:
+     * **Gzip (`[0x1F, 0x8B]`)**: `flate2::read::MultiGzDecoder` (`1.1.10`).
+     * **Zstandard (`[0x28, 0xB5, 0x2F, 0xFD]`)**: `zstd::stream::decode_all` (`0.14.3`).
+     * **LZ4 Frame (`[0x04, 0x22, 0x4D, 0x18]`)**: `lz4_flex::frame::FrameDecoder` (`0.14.0`, protected against `CVE-2026-32829`).
+
+3. **Full Arrow Type Extraction & Nested JSON Paths (`data/src/multimodal.rs` & `data/src/postprocess.rs`)**:
+   - `MultimodalValue::List(Vec<MultimodalValue>)` recursively extracts `ListArray`, `LargeListArray`, and `FixedSizeListArray` (supporting multi-turn chat `messages` and embedding vectors).
+   - `Float16Array` values are converted losslessly via `half::f16::to_f64()`.
+   - `extract_json_value_by_path` supports nested dot-paths and array indices (e.g., `json_extract_key="messages.0.content"` or `"meta.quality.score"`).
+
+4. **Typo Diagnostic Engine & Developer-Friendly Errors (`data/src/error.rs` & `python/luminamoon/data/diagnostics.py`)**:
+   - **Damerau-Levenshtein Optimal String Alignment (`damerau_levenshtein`, `find_closest_matches`)**: Computes edit distance including adjacent character transpositions (e.g., `"txet"` $\rightarrow$ `"text"`, `"prmopt"` $\rightarrow$ `"prompt"`).
+   - **Fake File / Git LFS Pointer Detection (`inspect_magic_bytes_hint`)**: Detects when a `.parquet`/`.arrow` file that fails to open is actually an unfetched Git LFS text pointer (`version https://git-lfs.github.com/spec/v1`), an HTML error page (`<!DOCTYPE html>`), or a misnamed JSON/Gzip/Zstd file.
+   - **Sibling Filename & URI Suggestions (`open_local_file_with_diagnostics`, `diagnose_uri_typo`)**: Suggests similar filenames in the same directory on `FileNotFound`, corrects URI scheme typos (`hf://dataset/...` $\rightarrow$ `hf://datasets/...`, `httpss://`), and validates unknown Python `**kwargs` with closest-parameter suggestions.

data/docs/library_audit/README.md

@@ -1,30 +1,30 @@
-# Library Version & Changelog Audit Documentation (October 2026)
-
-This directory documents the version-by-version changelog audit, intermediate API/ABI deprecations, security advisory mitigations (`RUSTSEC` / `CVE`), and source-code adaptations to the latest stable releases across all external libraries used by the **LuminaMoon Data Engine (`luminamoon-dataloader v1.1.0`)** crate.
-
----
-
-## Library Summary
-
-| Component / Library | Audited Version Range | Latest Stable Version | Adaptation Status | Detailed Document |
-| :--- | :--- | :--- | :--- | :--- |
-| **Rust Toolchain & `std::arch`** | `1.85.0` - `1.99.1` | `1.99.1` (`edition = "2024"`) | Adopted `std::cfg_select!`, `[T]::as_chunks::<3>()`, and `unsafe_op_in_unsafe_fn` compliance across `AVX-512BW` / `AVX2` / `NEON` intrinsics | [01_rust_toolchain_simd_and_mmap.md](01_rust_toolchain_simd_and_mmap.md) |
-| **System & Parallelism Crates** | `2024` - `2026` | `memmap2 0.9.11`, `rayon 1.12.0`, `thiserror 2.0.21`, `serde 1.0.229`, `serde_json 1.0.154` | Mitigated `RUSTSEC-2026-0186` (`memmap2`), indexed parallel batch processing via `rayon`, and direct byte-buffer streaming via `serde_json::to_writer` | [01_rust_toolchain_simd_and_mmap.md](01_rust_toolchain_simd_and_mmap.md) |
-| **Python, PyO3 & Maturin** | `Py 3.10` - `3.15.0`, `pyo3 0.24` - `0.30.1` | `Python 3.15.0`, `pyo3 0.30.1`, `maturin 1.15.0` | Removed `extension-module`, enabled `abi3t`, migrated to `py.detach`, `Bound::cast::<T>()`, and `#[pyclass(frozen, subclass)]` | [02_python_pyo3_abi3t_and_maturin.md](02_python_pyo3_abi3t_and_maturin.md) |
-| **Apache Arrow & Parquet** | `53.0.0` - `60.0.0` | `parquet 60.0.0`, `arrow-array 60.0.0`, `arrow-schema 60.0.0` | Added `StringViewArray`/`BinaryViewArray` support, `ArrowReaderMetadata` caching, and `&mut *self` reborrowing in `ParquetMetaDataReader` | [03_parquet_arrow_and_http206_streaming.md](03_parquet_arrow_and_http206_streaming.md) |
-| **Async Networking & HTTP Buffers** | `reqwest 0.12` - `0.13.5`, `tokio 1.38` - `1.53.3`, `bytes 1.6` - `1.12.1` | `reqwest 0.13.5`, `tokio 1.53.3`, `bytes 1.12.1`, `futures 0.3.34` | Mitigated `CVE-2026-25541` (`bytes 1.12.1`), explicit `default-tls` + `stream` + `json` feature selection on `reqwest 0.13.5` | [03_parquet_arrow_and_http206_streaming.md](03_parquet_arrow_and_http206_streaming.md) |
-| **Hashing, Tables & Synchronization** | `2024` - `2026` | `xxhash-rust 0.8.19`, `hashbrown 0.17.1`, `crc32fast 1.5.2`, `parking_lot 0.12.5` | Bypassed default `foldhash` in `hashbrown 0.17` via `IdentityBuildHasher`, enabled `vpclmulqdq` hardware acceleration in `crc32fast`, and enforced HuggingFace Hub NDJSON security | [04_xxh3_dedup_crc32_and_hf_hub_security.md](04_xxh3_dedup_crc32_and_hf_hub_security.md) |
-| **Universal Formats & Decompression** | `2025` - `2026` | `arrow-ipc 60.0.0`, `half 2.7.1`, `flate2 1.1.10`, `zstd 0.14.3`, `lz4_flex 0.14.0` | Mitigated `CVE-2026-32829` (`lz4_flex 0.14.0`), `DCtx` soundness fixes (`zstd 0.14.3`), `runtime_detection` (`flate2 1.1.10`), Arrow IPC/Feather/WebDataset TAR, and Damerau-Levenshtein typo diagnostics | [05_universal_formats_compression_and_diagnostics.md](05_universal_formats_compression_and_diagnostics.md) |
-
----
-
-## File Navigation Map
-
-| File Path | Role Summary | Status |
-| :--- | :--- | :--- |
-| `data/docs/library_audit/README.md` | Master index and version summary for all external libraries in `luminamoon-dataloader`. | Active |
-| `data/docs/library_audit/01_rust_toolchain_simd_and_mmap.md` | Changelog audit for Rust 1.85-1.99.1 (`std::arch`), `memmap2 0.9.11`, `rayon 1.12.0`, `thiserror 2.0.21`, and `serde`. | Active |
-| `data/docs/library_audit/02_python_pyo3_abi3t_and_maturin.md` | Changelog audit for Python 3.10-3.15, PyO3 0.24-0.30.1 (`abi3t`, `py.detach`, `Bound::cast`), and Maturin 1.15. | Active |
-| `data/docs/library_audit/03_parquet_arrow_and_http206_streaming.md` | Changelog audit for Apache Arrow/Parquet 53-60.0.0, `reqwest 0.13.5`, `tokio 1.53.3`, and `bytes 1.12.1`. | Active |
-| `data/docs/library_audit/04_xxh3_dedup_crc32_and_hf_hub_security.md` | Changelog audit for `xxhash-rust 0.8.19`, `hashbrown 0.17.1`, `crc32fast 1.5.2`, `parking_lot 0.12.5`, and HuggingFace Hub API specs. | Active |
-| `data/docs/library_audit/05_universal_formats_compression_and_diagnostics.md` | Version-by-version changelog audit (2025-2026) for `arrow-ipc 60.0.0`, `half 2.7.1`, `flate2 1.1.10`, `zstd 0.14.3`, `lz4_flex 0.14.0`, WebDataset `.tar` parser, and typo diagnostics. | Active |
+# Library Version & Changelog Audit Documentation (October 2026)
+
+This directory documents the version-by-version changelog audit, intermediate API/ABI deprecations, security advisory mitigations (`RUSTSEC` / `CVE`), and source-code adaptations to the latest stable releases across all external libraries used by the **LuminaMoon Data Engine (`luminamoon-dataloader v1.1.0`)** crate.
+
+---
+
+## Library Summary
+
+| Component / Library | Audited Version Range | Latest Stable Version | Adaptation Status | Detailed Document |
+| :--- | :--- | :--- | :--- | :--- |
+| **Rust Toolchain & `std::arch`** | `1.85.0` - `1.99.1` | `1.99.1` (`edition = "2024"`) | Adopted `std::cfg_select!`, `[T]::as_chunks::<3>()`, and `unsafe_op_in_unsafe_fn` compliance across `AVX-512BW` / `AVX2` / `NEON` intrinsics | [01_rust_toolchain_simd_and_mmap.md](01_rust_toolchain_simd_and_mmap.md) |
+| **System & Parallelism Crates** | `2024` - `2026` | `memmap2 0.9.11`, `rayon 1.12.0`, `thiserror 2.0.21`, `serde 1.0.229`, `serde_json 1.0.154` | Mitigated `RUSTSEC-2026-0186` (`memmap2`), indexed parallel batch processing via `rayon`, and direct byte-buffer streaming via `serde_json::to_writer` | [01_rust_toolchain_simd_and_mmap.md](01_rust_toolchain_simd_and_mmap.md) |
+| **Python, PyO3 & Maturin** | `Py 3.10` - `3.15.0`, `pyo3 0.24` - `0.30.1` | `Python 3.15.0`, `pyo3 0.30.1`, `maturin 1.15.0` | Removed `extension-module`, enabled `abi3t`, migrated to `py.detach`, `Bound::cast::<T>()`, and `#[pyclass(frozen, subclass)]` | [02_python_pyo3_abi3t_and_maturin.md](02_python_pyo3_abi3t_and_maturin.md) |
+| **Apache Arrow & Parquet** | `53.0.0` - `60.0.0` | `parquet 60.0.0`, `arrow-array 60.0.0`, `arrow-schema 60.0.0` | Added `StringViewArray`/`BinaryViewArray` support, `ArrowReaderMetadata` caching, and `&mut *self` reborrowing in `ParquetMetaDataReader` | [03_parquet_arrow_and_http206_streaming.md](03_parquet_arrow_and_http206_streaming.md) |
+| **Async Networking & HTTP Buffers** | `reqwest 0.12` - `0.13.5`, `tokio 1.38` - `1.53.3`, `bytes 1.6` - `1.12.1` | `reqwest 0.13.5`, `tokio 1.53.3`, `bytes 1.12.1`, `futures 0.3.34` | Mitigated `CVE-2026-25541` (`bytes 1.12.1`), explicit `default-tls` + `stream` + `json` feature selection on `reqwest 0.13.5` | [03_parquet_arrow_and_http206_streaming.md](03_parquet_arrow_and_http206_streaming.md) |
+| **Hashing, Tables & Synchronization** | `2024` - `2026` | `xxhash-rust 0.8.19`, `hashbrown 0.17.1`, `crc32fast 1.5.2`, `parking_lot 0.12.5` | Bypassed default `foldhash` in `hashbrown 0.17` via `IdentityBuildHasher`, enabled `vpclmulqdq` hardware acceleration in `crc32fast`, and enforced HuggingFace Hub NDJSON security | [04_xxh3_dedup_crc32_and_hf_hub_security.md](04_xxh3_dedup_crc32_and_hf_hub_security.md) |
+| **Universal Formats & Decompression** | `2025` - `2026` | `arrow-ipc 60.0.0`, `half 2.7.1`, `flate2 1.1.10`, `zstd 0.14.3`, `lz4_flex 0.14.0` | Mitigated `CVE-2026-32829` (`lz4_flex 0.14.0`), `DCtx` soundness fixes (`zstd 0.14.3`), `runtime_detection` (`flate2 1.1.10`), Arrow IPC/Feather/WebDataset TAR, and Damerau-Levenshtein typo diagnostics | [05_universal_formats_compression_and_diagnostics.md](05_universal_formats_compression_and_diagnostics.md) |
+
+---
+
+## File Navigation Map
+
+| File Path | Role Summary | Status |
+| :--- | :--- | :--- |
+| `data/docs/library_audit/README.md` | Master index and version summary for all external libraries in `luminamoon-dataloader`. | Active |
+| `data/docs/library_audit/01_rust_toolchain_simd_and_mmap.md` | Changelog audit for Rust 1.85-1.99.1 (`std::arch`), `memmap2 0.9.11`, `rayon 1.12.0`, `thiserror 2.0.21`, and `serde`. | Active |
+| `data/docs/library_audit/02_python_pyo3_abi3t_and_maturin.md` | Changelog audit for Python 3.10-3.15, PyO3 0.24-0.30.1 (`abi3t`, `py.detach`, `Bound::cast`), and Maturin 1.15. | Active |
+| `data/docs/library_audit/03_parquet_arrow_and_http206_streaming.md` | Changelog audit for Apache Arrow/Parquet 53-60.0.0, `reqwest 0.13.5`, `tokio 1.53.3`, and `bytes 1.12.1`. | Active |
+| `data/docs/library_audit/04_xxh3_dedup_crc32_and_hf_hub_security.md` | Changelog audit for `xxhash-rust 0.8.19`, `hashbrown 0.17.1`, `crc32fast 1.5.2`, `parking_lot 0.12.5`, and HuggingFace Hub API specs. | Active |
+| `data/docs/library_audit/05_universal_formats_compression_and_diagnostics.md` | Version-by-version changelog audit (2025-2026) for `arrow-ipc 60.0.0`, `half 2.7.1`, `flate2 1.1.10`, `zstd 0.14.3`, `lz4_flex 0.14.0`, WebDataset `.tar` parser, and typo diagnostics. | Active |

data/src/constants.rs

@@ -1,21 +1,21 @@
-pub const STREAM_COORD_BYTE_LEN: usize = 32;
-pub const STREAM_COORDINATE_BYTE_LEN: usize = STREAM_COORD_BYTE_LEN;
-
-pub const CHECKPOINT_SLOT_MAGIC: [u8; 4] = *b"LMDL";
-pub const META_SLOT_MAGIC: [u8; 4] = CHECKPOINT_SLOT_MAGIC;
-pub const CHECKPOINT_SLOT_VERSION: u32 = 1;
-pub const META_SLOT_VERSION: u32 = CHECKPOINT_SLOT_VERSION;
-pub const CHECKPOINT_SLOT_SIZE: usize = 64;
-pub const META_SLOT_SIZE: usize = CHECKPOINT_SLOT_SIZE;
-pub const CHECKPOINT_TOTAL_META_SIZE: u64 = (CHECKPOINT_SLOT_SIZE * 2) as u64;
-pub const TOTAL_META_SIZE: u64 = CHECKPOINT_TOTAL_META_SIZE;
-pub const DEDUP_FILE_SUFFIX: &str = ".dedup";
-
-pub const PARQUET_MAGIC_BYTES: &[u8; 4] = b"PAR1";
-pub const MAGIC_PARQUET: &[u8; 4] = PARQUET_MAGIC_BYTES;
-pub const ARROW_IPC_MAGIC_BYTES: &[u8; 6] = b"ARROW1";
-pub const MAGIC_ARROW_IPC: &[u8; 6] = ARROW_IPC_MAGIC_BYTES;
-
-pub const DEFAULT_BATCH_SIZE: usize = 64;
-pub const MAX_TYPO_SUGGESTIONS: usize = 3;
-pub const MAX_COLUMN_PREVIEW_COUNT: usize = 16;
+pub const STREAM_COORD_BYTE_LEN: usize = 32;
+pub const STREAM_COORDINATE_BYTE_LEN: usize = STREAM_COORD_BYTE_LEN;
+
+pub const CHECKPOINT_SLOT_MAGIC: [u8; 4] = *b"LMDL";
+pub const META_SLOT_MAGIC: [u8; 4] = CHECKPOINT_SLOT_MAGIC;
+pub const CHECKPOINT_SLOT_VERSION: u32 = 1;
+pub const META_SLOT_VERSION: u32 = CHECKPOINT_SLOT_VERSION;
+pub const CHECKPOINT_SLOT_SIZE: usize = 64;
+pub const META_SLOT_SIZE: usize = CHECKPOINT_SLOT_SIZE;
+pub const CHECKPOINT_TOTAL_META_SIZE: u64 = (CHECKPOINT_SLOT_SIZE * 2) as u64;
+pub const TOTAL_META_SIZE: u64 = CHECKPOINT_TOTAL_META_SIZE;
+pub const DEDUP_FILE_SUFFIX: &str = ".dedup";
+
+pub const PARQUET_MAGIC_BYTES: &[u8; 4] = b"PAR1";
+pub const MAGIC_PARQUET: &[u8; 4] = PARQUET_MAGIC_BYTES;
+pub const ARROW_IPC_MAGIC_BYTES: &[u8; 6] = b"ARROW1";
+pub const MAGIC_ARROW_IPC: &[u8; 6] = ARROW_IPC_MAGIC_BYTES;
+
+pub const DEFAULT_BATCH_SIZE: usize = 64;
+pub const MAX_TYPO_SUGGESTIONS: usize = 3;
+pub const MAX_COLUMN_PREVIEW_COUNT: usize = 16;

data/src/coord.rs

@@ -1,39 +1,39 @@
-use crate::constants::STREAM_COORDINATE_BYTE_LEN;
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
-pub struct StreamCoordinate {
-    pub shard_id: u32,
-    pub byte_offset: u64,
-    pub row_group_idx: u32,
-    pub page_idx: u32,
-    pub row_offset_in_page: u32,
-    pub global_row_idx: u64,
-}
-
-impl StreamCoordinate {
-    pub const BYTE_LEN: usize = STREAM_COORDINATE_BYTE_LEN;
-
-    #[inline]
-    pub fn to_bytes(&self) -> [u8; Self::BYTE_LEN] {
-        let mut buf = [0u8; Self::BYTE_LEN];
-        buf[0..4].copy_from_slice(&self.shard_id.to_le_bytes());
-        buf[4..12].copy_from_slice(&self.byte_offset.to_le_bytes());
-        buf[12..16].copy_from_slice(&self.row_group_idx.to_le_bytes());
-        buf[16..20].copy_from_slice(&self.page_idx.to_le_bytes());
-        buf[20..24].copy_from_slice(&self.row_offset_in_page.to_le_bytes());
-        buf[24..32].copy_from_slice(&self.global_row_idx.to_le_bytes());
-        buf
-    }
-
-    #[inline]
-    pub fn from_bytes(buf: &[u8; Self::BYTE_LEN]) -> Self {
-        Self {
-            shard_id: u32::from_le_bytes(buf[0..4].try_into().unwrap()),
-            byte_offset: u64::from_le_bytes(buf[4..12].try_into().unwrap()),
-            row_group_idx: u32::from_le_bytes(buf[12..16].try_into().unwrap()),
-            page_idx: u32::from_le_bytes(buf[16..20].try_into().unwrap()),
-            row_offset_in_page: u32::from_le_bytes(buf[20..24].try_into().unwrap()),
-            global_row_idx: u64::from_le_bytes(buf[24..32].try_into().unwrap()),
-        }
-    }
-}
+use crate::constants::STREAM_COORDINATE_BYTE_LEN;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+pub struct StreamCoordinate {
+    pub shard_id: u32,
+    pub byte_offset: u64,
+    pub row_group_idx: u32,
+    pub page_idx: u32,
+    pub row_offset_in_page: u32,
+    pub global_row_idx: u64,
+}
+
+impl StreamCoordinate {
+    pub const BYTE_LEN: usize = STREAM_COORDINATE_BYTE_LEN;
+
+    #[inline]
+    pub fn to_bytes(&self) -> [u8; Self::BYTE_LEN] {
+        let mut buf = [0u8; Self::BYTE_LEN];
+        buf[0..4].copy_from_slice(&self.shard_id.to_le_bytes());
+        buf[4..12].copy_from_slice(&self.byte_offset.to_le_bytes());
+        buf[12..16].copy_from_slice(&self.row_group_idx.to_le_bytes());
+        buf[16..20].copy_from_slice(&self.page_idx.to_le_bytes());
+        buf[20..24].copy_from_slice(&self.row_offset_in_page.to_le_bytes());
+        buf[24..32].copy_from_slice(&self.global_row_idx.to_le_bytes());
+        buf
+    }
+
+    #[inline]
+    pub fn from_bytes(buf: &[u8; Self::BYTE_LEN]) -> Self {
+        Self {
+            shard_id: u32::from_le_bytes(buf[0..4].try_into().unwrap()),
+            byte_offset: u64::from_le_bytes(buf[4..12].try_into().unwrap()),
+            row_group_idx: u32::from_le_bytes(buf[12..16].try_into().unwrap()),
+            page_idx: u32::from_le_bytes(buf[16..20].try_into().unwrap()),
+            row_offset_in_page: u32::from_le_bytes(buf[20..24].try_into().unwrap()),
+            global_row_idx: u64::from_le_bytes(buf[24..32].try_into().unwrap()),
+        }
+    }
+}

data/src/dedup.rs

@@ -1,112 +1,112 @@
-use std::hash::{BuildHasherDefault, Hasher};
-
-use hashbrown::{HashMap, HashSet};
-use xxhash_rust::xxh3::xxh3_64;
-
-#[derive(Default)]
-pub struct IdentityHasher(u64);
-
-impl Hasher for IdentityHasher {
-    #[inline]
-    fn finish(&self) -> u64 {
-        self.0
-    }
-
-    #[inline]
-    fn write(&mut self, bytes: &[u8]) {
-        self.0 = xxh3_64(bytes);
-    }
-
-    #[inline]
-    fn write_u64(&mut self, i: u64) {
-        self.0 = i;
-    }
-}
-
-pub type IdentityBuildHasher = BuildHasherDefault<IdentityHasher>;
-
-#[derive(Debug, Default)]
-pub struct TwoStageDeduplicator {
-    committed_hashes: HashSet<u64, IdentityBuildHasher>,
-    staged_lookup: HashSet<u64, IdentityBuildHasher>,
-    staged_batches: HashMap<u64, Vec<u64>>,
-}
-
-impl TwoStageDeduplicator {
-    pub fn new() -> Self {
-        Self::default()
-    }
-
-    pub fn with_committed(hashes: impl IntoIterator<Item = u64>) -> Self {
-        let committed_hashes: HashSet<u64, IdentityBuildHasher> = hashes.into_iter().collect();
-        Self {
-            committed_hashes,
-            staged_lookup: HashSet::default(),
-            staged_batches: HashMap::new(),
-        }
-    }
-
-    #[inline]
-    pub fn compute_hash(payload: &[u8]) -> u64 {
-        xxh3_64(payload)
-    }
-
-    #[inline]
-    pub fn is_seen(&self, payload: &[u8]) -> bool {
-        let hash = Self::compute_hash(payload);
-        self.committed_hashes.contains(&hash) || self.staged_lookup.contains(&hash)
-    }
-
-    #[inline]
-    pub fn try_stage(&mut self, batch_id: u64, payload: &[u8]) -> Option<u64> {
-        let hash = Self::compute_hash(payload);
-        if self.committed_hashes.contains(&hash) || self.staged_lookup.contains(&hash) {
-            return None;
-        }
-        self.staged_lookup.insert(hash);
-        self.staged_batches.entry(batch_id).or_default().push(hash);
-        Some(hash)
-    }
-
-    pub fn commit_batch(&mut self, batch_id: u64) -> Vec<u64> {
-        let Some(hashes) = self.staged_batches.remove(&batch_id) else {
-            return Vec::new();
-        };
-        for &h in &hashes {
-            self.staged_lookup.remove(&h);
-            self.committed_hashes.insert(h);
-        }
-        hashes
-    }
-
-    pub fn rollback_batch(&mut self, batch_id: u64) -> usize {
-        let Some(hashes) = self.staged_batches.remove(&batch_id) else {
-            return 0;
-        };
-        for &h in &hashes {
-            self.staged_lookup.remove(&h);
-        }
-        hashes.len()
-    }
-
-    pub fn clear_staged(&mut self) {
-        self.staged_lookup.clear();
-        self.staged_batches.clear();
-    }
-
-    pub fn committed_len(&self) -> usize {
-        self.committed_hashes.len()
-    }
-
-    pub fn staged_len(&self) -> usize {
-        self.staged_lookup.len()
-    }
-
-    pub fn export_committed_hashes(&self) -> Vec<u64> {
-        self.committed_hashes.iter().copied().collect()
-    }
-
-    pub fn import_committed_hashes(&mut self, hashes: impl IntoIterator<Item = u64>) {
-        self.committed_hashes.extend(hashes);
-    }
-}
+use std::hash::{BuildHasherDefault, Hasher};
+
+use hashbrown::{HashMap, HashSet};
+use xxhash_rust::xxh3::xxh3_64;
+
+#[derive(Default)]
+pub struct IdentityHasher(u64);
+
+impl Hasher for IdentityHasher {
+    #[inline]
+    fn finish(&self) -> u64 {
+        self.0
+    }
+
+    #[inline]
+    fn write(&mut self, bytes: &[u8]) {
+        self.0 = xxh3_64(bytes);
+    }
+
+    #[inline]
+    fn write_u64(&mut self, i: u64) {
+        self.0 = i;
+    }
+}
+
+pub type IdentityBuildHasher = BuildHasherDefault<IdentityHasher>;
+
+#[derive(Debug, Default)]
+pub struct TwoStageDeduplicator {
+    committed_hashes: HashSet<u64, IdentityBuildHasher>,
+    staged_lookup: HashSet<u64, IdentityBuildHasher>,
+    staged_batches: HashMap<u64, Vec<u64>>,
+}
+
+impl TwoStageDeduplicator {
+    pub fn new() -> Self {
+        Self::default()
+    }
+
+    pub fn with_committed(hashes: impl IntoIterator<Item = u64>) -> Self {
+        let committed_hashes: HashSet<u64, IdentityBuildHasher> = hashes.into_iter().collect();
+        Self {
+            committed_hashes,
+            staged_lookup: HashSet::default(),
+            staged_batches: HashMap::new(),
+        }
+    }
+
+    #[inline]
+    pub fn compute_hash(payload: &[u8]) -> u64 {
+        xxh3_64(payload)
+    }
+
+    #[inline]
+    pub fn is_seen(&self, payload: &[u8]) -> bool {
+        let hash = Self::compute_hash(payload);
+        self.committed_hashes.contains(&hash) || self.staged_lookup.contains(&hash)
+    }
+
+    #[inline]
+    pub fn try_stage(&mut self, batch_id: u64, payload: &[u8]) -> Option<u64> {
+        let hash = Self::compute_hash(payload);
+        if self.committed_hashes.contains(&hash) || self.staged_lookup.contains(&hash) {
+            return None;
+        }
+        self.staged_lookup.insert(hash);
+        self.staged_batches.entry(batch_id).or_default().push(hash);
+        Some(hash)
+    }
+
+    pub fn commit_batch(&mut self, batch_id: u64) -> Vec<u64> {
+        let Some(hashes) = self.staged_batches.remove(&batch_id) else {
+            return Vec::new();
+        };
+        for &h in &hashes {
+            self.staged_lookup.remove(&h);
+            self.committed_hashes.insert(h);
+        }
+        hashes
+    }
+
+    pub fn rollback_batch(&mut self, batch_id: u64) -> usize {
+        let Some(hashes) = self.staged_batches.remove(&batch_id) else {
+            return 0;
+        };
+        for &h in &hashes {
+            self.staged_lookup.remove(&h);
+        }
+        hashes.len()
+    }
+
+    pub fn clear_staged(&mut self) {
+        self.staged_lookup.clear();
+        self.staged_batches.clear();
+    }
+
+    pub fn committed_len(&self) -> usize {
+        self.committed_hashes.len()
+    }
+
+    pub fn staged_len(&self) -> usize {
+        self.staged_lookup.len()
+    }
+
+    pub fn export_committed_hashes(&self) -> Vec<u64> {
+        self.committed_hashes.iter().copied().collect()
+    }
+
+    pub fn import_committed_hashes(&mut self, hashes: impl IntoIterator<Item = u64>) {
+        self.committed_hashes.extend(hashes);
+    }
+}

data/src/error.rs

@@ -1,166 +1,166 @@
-use thiserror::Error;
-
-use crate::constants::{MAX_COLUMN_PREVIEW_COUNT, MAX_TYPO_SUGGESTIONS};
-
-#[derive(Debug, Error)]
-pub enum DataLoaderError {
-    #[error("I/O failure: {0}")]
-    Io(#[from] std::io::Error),
-
-    #[error("Parquet failure: {0}")]
-    Parquet(#[from] parquet::errors::ParquetError),
-
-    #[error("Arrow failure: {0}")]
-    Arrow(#[from] arrow_schema::ArrowError),
-
-    #[error("Corrupted metadata checkpoint in both slots")]
-    CorruptedMetadata,
-
-    #[error("Unknown staged batch id: {0}")]
-    UnknownBatchId(u64),
-
-    #[error("Invalid coordinate buffer length")]
-    InvalidCoordinate,
-
-    #[error("Target column not found: {0}")]
-    ColumnNotFound(String),
-
-    #[error("Invalid configuration: {0}")]
-    InvalidConfig(String),
-}
-
-pub type Result<T> = std::result::Result<T, DataLoaderError>;
-
-pub fn damerau_levenshtein(a: &str, b: &str) -> usize {
-    let a_chars: Vec<char> = a.chars().collect();
-    let b_chars: Vec<char> = b.chars().collect();
-    let len_a = a_chars.len();
-    let len_b = b_chars.len();
-
-    if len_a == 0 {
-        return len_b;
-    }
-    if len_b == 0 {
-        return len_a;
-    }
-
-    let width = len_b + 1;
-    let mut matrix = vec![0usize; (len_a + 1) * width];
-
-    for i in 0..=len_a {
-        matrix[i * width] = i;
-    }
-    for (j, cell) in matrix.iter_mut().enumerate().take(len_b + 1) {
-        *cell = j;
-    }
-
-    for i in 1..=len_a {
-        for j in 1..=len_b {
-            let cost = usize::from(a_chars[i - 1] != b_chars[j - 1]);
-            let deletion = matrix[(i - 1) * width + j] + 1;
-            let insertion = matrix[i * width + (j - 1)] + 1;
-            let substitution = matrix[(i - 1) * width + (j - 1)] + cost;
-            let mut best = deletion.min(insertion).min(substitution);
-
-            if i > 1
-                && j > 1
-                && a_chars[i - 1] == b_chars[j - 2]
-                && a_chars[i - 2] == b_chars[j - 1]
-            {
-                let transposition = matrix[(i - 2) * width + (j - 2)] + 1;
-                best = best.min(transposition);
-            }
-            matrix[i * width + j] = best;
-        }
-    }
-
-    matrix[len_a * width + len_b]
-}
-
-pub fn find_closest_matches<S: AsRef<str>>(
-    target: &str,
-    candidates: &[S],
-    max_suggestions: usize,
-) -> Vec<String> {
-    let target_trim = target.trim();
-    if target_trim.is_empty() || candidates.is_empty() || max_suggestions == 0 {
-        return Vec::new();
-    }
-    let target_lower = target_trim.to_ascii_lowercase();
-
-    let mut scored: Vec<(usize, usize, String)> = Vec::new();
-    for cand in candidates {
-        let c_str = cand.as_ref();
-        if c_str.is_empty() {
-            continue;
-        }
-        let c_lower = c_str.to_ascii_lowercase();
-        if c_lower == target_lower {
-            scored.push((0, 0, c_str.to_owned()));
-            continue;
-        }
-        let dist = damerau_levenshtein(&target_lower, &c_lower);
-        let max_len = target_lower.chars().count().max(c_lower.chars().count());
-        let threshold = match max_len {
-            0..=3 => 1,
-            4..=6 => 2,
-            7..=11 => 3,
-            _ => (max_len * 2) / 5,
-        };
-        let is_sub = c_lower.contains(&target_lower) || target_lower.contains(&c_lower);
-        if dist <= threshold || (target_lower.len() >= 3 && is_sub) {
-            let rank_bonus = if is_sub { 0 } else { 1 };
-            scored.push((dist, rank_bonus, c_str.to_owned()));
-        }
-    }
-
-    scored.sort_by(|a, b| a.0.cmp(&b.0).then(a.1.cmp(&b.1)).then(a.2.cmp(&b.2)));
-    scored.dedup_by(|a, b| a.2 == b.2);
-    scored
-        .into_iter()
-        .take(max_suggestions)
-        .map(|(_, _, s)| s)
-        .collect()
-}
-
-pub fn format_did_you_mean<S: AsRef<str>>(target: &str, candidates: &[S]) -> String {
-    let matches = find_closest_matches(target, candidates, MAX_TYPO_SUGGESTIONS);
-    if matches.is_empty() {
-        return String::new();
-    }
-    if matches.len() == 1 {
-        format!(" Did you mean '{}'?", matches[0])
-    } else {
-        let quoted: Vec<String> = matches.iter().map(|m| format!("'{m}'")).collect();
-        format!(" Did you mean one of [{}]?", quoted.join(", "))
-    }
-}
-
-pub fn column_not_found_with_suggestions<S: AsRef<str>>(
-    requested: &str,
-    available: &[S],
-) -> DataLoaderError {
-    let suggestion = format_did_you_mean(requested, available);
-    let avail_preview: Vec<String> = available
-        .iter()
-        .take(MAX_COLUMN_PREVIEW_COUNT)
-        .map(|s| format!("'{}'", s.as_ref()))
-        .collect();
-    let avail_suffix = if available.len() > MAX_COLUMN_PREVIEW_COUNT {
-        format!(
-            " (and {} more)",
-            available.len().saturating_sub(avail_preview.len())
-        )
-    } else {
-        String::new()
-    };
-
-    if avail_preview.is_empty() {
-        DataLoaderError::ColumnNotFound(requested.to_owned())
-    } else {
-        DataLoaderError::ColumnNotFound(format!(
-            "'{requested}'.{suggestion} Available columns: [{}]{avail_suffix}",
-            avail_preview.join(", ")
-        ))
-    }
-}
+use thiserror::Error;
+
+use crate::constants::{MAX_COLUMN_PREVIEW_COUNT, MAX_TYPO_SUGGESTIONS};
+
+#[derive(Debug, Error)]
+pub enum DataLoaderError {
+    #[error("I/O failure: {0}")]
+    Io(#[from] std::io::Error),
+
+    #[error("Parquet failure: {0}")]
+    Parquet(#[from] parquet::errors::ParquetError),
+
+    #[error("Arrow failure: {0}")]
+    Arrow(#[from] arrow_schema::ArrowError),
+
+    #[error("Corrupted metadata checkpoint in both slots")]
+    CorruptedMetadata,
+
+    #[error("Unknown staged batch id: {0}")]
+    UnknownBatchId(u64),
+
+    #[error("Invalid coordinate buffer length")]
+    InvalidCoordinate,
+
+    #[error("Target column not found: {0}")]
+    ColumnNotFound(String),
+
+    #[error("Invalid configuration: {0}")]
+    InvalidConfig(String),
+}
+
+pub type Result<T> = std::result::Result<T, DataLoaderError>;
+
+pub fn damerau_levenshtein(a: &str, b: &str) -> usize {
+    let a_chars: Vec<char> = a.chars().collect();
+    let b_chars: Vec<char> = b.chars().collect();
+    let len_a = a_chars.len();
+    let len_b = b_chars.len();
+
+    if len_a == 0 {
+        return len_b;
+    }
+    if len_b == 0 {
+        return len_a;
+    }
+
+    let width = len_b + 1;
+    let mut matrix = vec![0usize; (len_a + 1) * width];
+
+    for i in 0..=len_a {
+        matrix[i * width] = i;
+    }
+    for (j, cell) in matrix.iter_mut().enumerate().take(len_b + 1) {
+        *cell = j;
+    }
+
+    for i in 1..=len_a {
+        for j in 1..=len_b {
+            let cost = usize::from(a_chars[i - 1] != b_chars[j - 1]);
+            let deletion = matrix[(i - 1) * width + j] + 1;
+            let insertion = matrix[i * width + (j - 1)] + 1;
+            let substitution = matrix[(i - 1) * width + (j - 1)] + cost;
+            let mut best = deletion.min(insertion).min(substitution);
+
+            if i > 1
+                && j > 1
+                && a_chars[i - 1] == b_chars[j - 2]
+                && a_chars[i - 2] == b_chars[j - 1]
+            {
+                let transposition = matrix[(i - 2) * width + (j - 2)] + 1;
+                best = best.min(transposition);
+            }
+            matrix[i * width + j] = best;
+        }
+    }
+
+    matrix[len_a * width + len_b]
+}
+
+pub fn find_closest_matches<S: AsRef<str>>(
+    target: &str,
+    candidates: &[S],
+    max_suggestions: usize,
+) -> Vec<String> {
+    let target_trim = target.trim();
+    if target_trim.is_empty() || candidates.is_empty() || max_suggestions == 0 {
+        return Vec::new();
+    }
+    let target_lower = target_trim.to_ascii_lowercase();
+
+    let mut scored: Vec<(usize, usize, String)> = Vec::new();
+    for cand in candidates {
+        let c_str = cand.as_ref();
+        if c_str.is_empty() {
+            continue;
+        }
+        let c_lower = c_str.to_ascii_lowercase();
+        if c_lower == target_lower {
+            scored.push((0, 0, c_str.to_owned()));
+            continue;
+        }
+        let dist = damerau_levenshtein(&target_lower, &c_lower);
+        let max_len = target_lower.chars().count().max(c_lower.chars().count());
+        let threshold = match max_len {
+            0..=3 => 1,
+            4..=6 => 2,
+            7..=11 => 3,
+            _ => (max_len * 2) / 5,
+        };
+        let is_sub = c_lower.contains(&target_lower) || target_lower.contains(&c_lower);
+        if dist <= threshold || (target_lower.len() >= 3 && is_sub) {
+            let rank_bonus = if is_sub { 0 } else { 1 };
+            scored.push((dist, rank_bonus, c_str.to_owned()));
+        }
+    }
+
+    scored.sort_by(|a, b| a.0.cmp(&b.0).then(a.1.cmp(&b.1)).then(a.2.cmp(&b.2)));
+    scored.dedup_by(|a, b| a.2 == b.2);
+    scored
+        .into_iter()
+        .take(max_suggestions)
+        .map(|(_, _, s)| s)
+        .collect()
+}
+
+pub fn format_did_you_mean<S: AsRef<str>>(target: &str, candidates: &[S]) -> String {
+    let matches = find_closest_matches(target, candidates, MAX_TYPO_SUGGESTIONS);
+    if matches.is_empty() {
+        return String::new();
+    }
+    if matches.len() == 1 {
+        format!(" Did you mean '{}'?", matches[0])
+    } else {
+        let quoted: Vec<String> = matches.iter().map(|m| format!("'{m}'")).collect();
+        format!(" Did you mean one of [{}]?", quoted.join(", "))
+    }
+}
+
+pub fn column_not_found_with_suggestions<S: AsRef<str>>(
+    requested: &str,
+    available: &[S],
+) -> DataLoaderError {
+    let suggestion = format_did_you_mean(requested, available);
+    let avail_preview: Vec<String> = available
+        .iter()
+        .take(MAX_COLUMN_PREVIEW_COUNT)
+        .map(|s| format!("'{}'", s.as_ref()))
+        .collect();
+    let avail_suffix = if available.len() > MAX_COLUMN_PREVIEW_COUNT {
+        format!(
+            " (and {} more)",
+            available.len().saturating_sub(avail_preview.len())
+        )
+    } else {
+        String::new()
+    };
+
+    if avail_preview.is_empty() {
+        DataLoaderError::ColumnNotFound(requested.to_owned())
+    } else {
+        DataLoaderError::ColumnNotFound(format!(
+            "'{requested}'.{suggestion} Available columns: [{}]{avail_suffix}",
+            avail_preview.join(", ")
+        ))
+    }
+}

data/src/lib.rs

@@ -1,15 +1,15 @@
-pub mod constants;
-pub mod coord;
-pub mod dedup;
-pub mod error;
-pub mod metadata;
-pub mod minhash;
-pub mod postprocess;
-pub mod py_bridge;
-pub mod quality;
-pub mod serialize;
-pub mod simd_cpu;
-pub mod string_tensor;
-
-#[cfg(test)]
-mod tests;
+pub mod constants;
+pub mod coord;
+pub mod dedup;
+pub mod error;
+pub mod metadata;
+pub mod minhash;
+pub mod postprocess;
+pub mod py_bridge;
+pub mod quality;
+pub mod serialize;
+pub mod simd_cpu;
+pub mod string_tensor;
+
+#[cfg(test)]
+mod tests;

data/src/metadata.rs

@@ -1,264 +1,264 @@
-use std::fs::{File, OpenOptions};
-use std::io::{Read, Seek, SeekFrom, Write};
-use std::path::{Path, PathBuf};
-
-use crc32fast::Hasher;
-use hashbrown::HashMap;
-
-use crate::constants::{
-    DEDUP_FILE_SUFFIX, META_SLOT_MAGIC, META_SLOT_SIZE, META_SLOT_VERSION, TOTAL_META_SIZE,
-};
-use crate::coord::StreamCoordinate;
-use crate::dedup::TwoStageDeduplicator;
-use crate::error::{DataLoaderError, Result};
-
-const SLOT_MAGIC: [u8; 4] = META_SLOT_MAGIC;
-const SLOT_VERSION: u32 = META_SLOT_VERSION;
-const SLOT_SIZE: usize = META_SLOT_SIZE;
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
-struct DiskSlot {
-    epoch: u64,
-    committed_batch_id: u64,
-    coord: StreamCoordinate,
-    dedup_count: u64,
-}
-
-impl DiskSlot {
-    fn encode(&self) -> [u8; SLOT_SIZE] {
-        let mut buf = [0u8; SLOT_SIZE];
-        buf[0..4].copy_from_slice(&SLOT_MAGIC);
-        buf[4..8].copy_from_slice(&SLOT_VERSION.to_le_bytes());
-        buf[8..16].copy_from_slice(&self.epoch.to_le_bytes());
-        buf[16..24].copy_from_slice(&self.committed_batch_id.to_le_bytes());
-        buf[24..56].copy_from_slice(&self.coord.to_bytes());
-        buf[56..60].copy_from_slice(&(self.dedup_count as u32).to_le_bytes());
-
-        let mut hasher = Hasher::new();
-        hasher.update(&buf[0..60]);
-        let crc = hasher.finalize();
-        buf[60..64].copy_from_slice(&crc.to_le_bytes());
-        buf
-    }
-
-    fn decode(buf: &[u8; SLOT_SIZE]) -> Option<Self> {
-        if buf[0..4] != SLOT_MAGIC {
-            return None;
-        }
-        let version = u32::from_le_bytes(buf[4..8].try_into().unwrap());
-        if version != SLOT_VERSION {
-            return None;
-        }
-        let expected_crc = u32::from_le_bytes(buf[60..64].try_into().unwrap());
-        let mut hasher = Hasher::new();
-        hasher.update(&buf[0..60]);
-        if hasher.finalize() != expected_crc {
-            return None;
-        }
-        let coord_bytes: &[u8; StreamCoordinate::BYTE_LEN] = buf[24..56].try_into().unwrap();
-        Some(Self {
-            epoch: u64::from_le_bytes(buf[8..16].try_into().unwrap()),
-            committed_batch_id: u64::from_le_bytes(buf[16..24].try_into().unwrap()),
-            coord: StreamCoordinate::from_bytes(coord_bytes),
-            dedup_count: u32::from_le_bytes(buf[56..60].try_into().unwrap()) as u64,
-        })
-    }
-}
-
-pub struct DualMetadataManager {
-    meta_file: File,
-    dedup_file: File,
-    epoch: u64,
-    committed_batch_id: u64,
-    committed_coord: StreamCoordinate,
-    in_flight_coord: StreamCoordinate,
-    staged_coords: HashMap<u64, StreamCoordinate>,
-    pub deduplicator: TwoStageDeduplicator,
-}
-
-impl DualMetadataManager {
-    pub fn open(meta_path: impl AsRef<Path>) -> Result<Self> {
-        let meta_path = meta_path.as_ref();
-        if let Some(parent) = meta_path.parent()
-            && !parent.as_os_str().is_empty()
-        {
-            std::fs::create_dir_all(parent)?;
-        }
-
-        let dedup_path = Self::dedup_path_for(meta_path);
-
-        let mut meta_file = OpenOptions::new()
-            .read(true)
-            .write(true)
-            .create(true)
-            .truncate(false)
-            .open(meta_path)?;
-
-        let mut dedup_file = OpenOptions::new()
-            .read(true)
-            .write(true)
-            .create(true)
-            .truncate(false)
-            .open(&dedup_path)?;
-
-        let file_len = meta_file.metadata()?.len();
-        let active_slot = if file_len < TOTAL_META_SIZE {
-            meta_file.set_len(TOTAL_META_SIZE)?;
-            let initial = DiskSlot::default();
-            let encoded = initial.encode();
-            meta_file.seek(SeekFrom::Start(0))?;
-            meta_file.write_all(&encoded)?;
-            meta_file.write_all(&encoded)?;
-            meta_file.sync_data()?;
-            initial
-        } else {
-            let mut raw = [0u8; SLOT_SIZE * 2];
-            meta_file.seek(SeekFrom::Start(0))?;
-            meta_file.read_exact(&mut raw)?;
-
-            let slot_a = DiskSlot::decode(raw[0..SLOT_SIZE].try_into().unwrap());
-            let slot_b = DiskSlot::decode(raw[SLOT_SIZE..SLOT_SIZE * 2].try_into().unwrap());
-
-            match (slot_a, slot_b) {
-                (Some(a), Some(b)) => {
-                    if a.epoch >= b.epoch {
-                        a
-                    } else {
-                        b
-                    }
-                }
-                (Some(a), None) => a,
-                (None, Some(b)) => b,
-                (None, None) => return Err(DataLoaderError::CorruptedMetadata),
-            }
-        };
-
-        let valid_dedup_bytes = active_slot.dedup_count * 8;
-        if dedup_file.metadata()?.len() < valid_dedup_bytes {
-            return Err(DataLoaderError::CorruptedMetadata);
-        }
-        dedup_file.set_len(valid_dedup_bytes)?;
-        dedup_file.seek(SeekFrom::Start(0))?;
-
-        let mut raw_hashes = vec![0u8; valid_dedup_bytes as usize];
-        if valid_dedup_bytes > 0 {
-            dedup_file.read_exact(&mut raw_hashes)?;
-        }
-
-        let restored_hashes = raw_hashes
-            .as_chunks::<8>()
-            .0
-            .iter()
-            .map(|&c| u64::from_le_bytes(c));
-
-        let deduplicator = TwoStageDeduplicator::with_committed(restored_hashes);
-
-        Ok(Self {
-            meta_file,
-            dedup_file,
-            epoch: active_slot.epoch,
-            committed_batch_id: active_slot.committed_batch_id,
-            committed_coord: active_slot.coord,
-            in_flight_coord: active_slot.coord,
-            staged_coords: HashMap::new(),
-            deduplicator,
-        })
-    }
-
-    fn dedup_path_for(meta_path: &Path) -> PathBuf {
-        let mut p = meta_path.as_os_str().to_owned();
-        p.push(DEDUP_FILE_SUFFIX);
-        PathBuf::from(p)
-    }
-
-    #[inline]
-    pub fn committed_coordinate(&self) -> StreamCoordinate {
-        self.committed_coord
-    }
-
-    #[inline]
-    pub fn in_flight_coordinate(&self) -> StreamCoordinate {
-        self.in_flight_coord
-    }
-
-    #[inline]
-    pub fn committed_batch_id(&self) -> u64 {
-        self.committed_batch_id
-    }
-
-    #[inline]
-    pub fn set_in_flight_coordinate(&mut self, next_coord: StreamCoordinate) {
-        self.in_flight_coord = next_coord;
-    }
-
-    pub fn stage_batch_in_ram(&mut self, batch_id: u64, next_coord: StreamCoordinate) {
-        self.in_flight_coord = next_coord;
-        self.staged_coords.insert(batch_id, next_coord);
-    }
-
-    pub fn commit_gpu_batch(&mut self, batch_id: u64) -> Result<StreamCoordinate> {
-        let Some(next_coord) = self.staged_coords.remove(&batch_id) else {
-            return Err(DataLoaderError::UnknownBatchId(batch_id));
-        };
-
-        let prev_dedup_count = self.deduplicator.committed_len() as u64;
-        let new_hashes = self.deduplicator.commit_batch(batch_id);
-        if !new_hashes.is_empty() {
-            let mut hash_buf = Vec::with_capacity(new_hashes.len() * 8);
-            for h in new_hashes {
-                hash_buf.extend_from_slice(&h.to_le_bytes());
-            }
-            let write_offset = prev_dedup_count * 8;
-            self.dedup_file.seek(SeekFrom::Start(write_offset))?;
-            self.dedup_file.write_all(&hash_buf)?;
-            self.dedup_file
-                .set_len(write_offset + hash_buf.len() as u64)?;
-            self.dedup_file.sync_data()?;
-        }
-
-        self.epoch += 1;
-        self.committed_batch_id = batch_id;
-        self.committed_coord = next_coord;
-
-        let slot = DiskSlot {
-            epoch: self.epoch,
-            committed_batch_id: self.committed_batch_id,
-            coord: self.committed_coord,
-            dedup_count: self.deduplicator.committed_len() as u64,
-        };
-
-        let slot_offset = if self.epoch % 2 == 1 {
-            0
-        } else {
-            SLOT_SIZE as u64
-        };
-
-        let encoded = slot.encode();
-        self.meta_file.seek(SeekFrom::Start(slot_offset))?;
-        self.meta_file.write_all(&encoded)?;
-        self.meta_file.sync_data()?;
-
-        Ok(self.committed_coord)
-    }
-
-    pub fn rollback_staged_batch(&mut self, batch_id: u64) -> Result<StreamCoordinate> {
-        if self.staged_coords.remove(&batch_id).is_none() {
-            return Err(DataLoaderError::UnknownBatchId(batch_id));
-        }
-        self.deduplicator.rollback_batch(batch_id);
-        self.in_flight_coord = self
-            .staged_coords
-            .iter()
-            .max_by_key(|&(&id, _)| id)
-            .map(|(_, &coord)| coord)
-            .unwrap_or(self.committed_coord);
-        Ok(self.in_flight_coord)
-    }
-
-    pub fn rollback_ram_to_committed(&mut self) -> StreamCoordinate {
-        self.staged_coords.clear();
-        self.deduplicator.clear_staged();
-        self.in_flight_coord = self.committed_coord;
-        self.committed_coord
-    }
-}
+use std::fs::{File, OpenOptions};
+use std::io::{Read, Seek, SeekFrom, Write};
+use std::path::{Path, PathBuf};
+
+use crc32fast::Hasher;
+use hashbrown::HashMap;
+
+use crate::constants::{
+    DEDUP_FILE_SUFFIX, META_SLOT_MAGIC, META_SLOT_SIZE, META_SLOT_VERSION, TOTAL_META_SIZE,
+};
+use crate::coord::StreamCoordinate;
+use crate::dedup::TwoStageDeduplicator;
+use crate::error::{DataLoaderError, Result};
+
+const SLOT_MAGIC: [u8; 4] = META_SLOT_MAGIC;
+const SLOT_VERSION: u32 = META_SLOT_VERSION;
+const SLOT_SIZE: usize = META_SLOT_SIZE;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+struct DiskSlot {
+    epoch: u64,
+    committed_batch_id: u64,
+    coord: StreamCoordinate,
+    dedup_count: u64,
+}
+
+impl DiskSlot {
+    fn encode(&self) -> [u8; SLOT_SIZE] {
+        let mut buf = [0u8; SLOT_SIZE];
+        buf[0..4].copy_from_slice(&SLOT_MAGIC);
+        buf[4..8].copy_from_slice(&SLOT_VERSION.to_le_bytes());
+        buf[8..16].copy_from_slice(&self.epoch.to_le_bytes());
+        buf[16..24].copy_from_slice(&self.committed_batch_id.to_le_bytes());
+        buf[24..56].copy_from_slice(&self.coord.to_bytes());
+        buf[56..60].copy_from_slice(&(self.dedup_count as u32).to_le_bytes());
+
+        let mut hasher = Hasher::new();
+        hasher.update(&buf[0..60]);
+        let crc = hasher.finalize();
+        buf[60..64].copy_from_slice(&crc.to_le_bytes());
+        buf
+    }
+
+    fn decode(buf: &[u8; SLOT_SIZE]) -> Option<Self> {
+        if buf[0..4] != SLOT_MAGIC {
+            return None;
+        }
+        let version = u32::from_le_bytes(buf[4..8].try_into().unwrap());
+        if version != SLOT_VERSION {
+            return None;
+        }
+        let expected_crc = u32::from_le_bytes(buf[60..64].try_into().unwrap());
+        let mut hasher = Hasher::new();
+        hasher.update(&buf[0..60]);
+        if hasher.finalize() != expected_crc {
+            return None;
+        }
+        let coord_bytes: &[u8; StreamCoordinate::BYTE_LEN] = buf[24..56].try_into().unwrap();
+        Some(Self {
+            epoch: u64::from_le_bytes(buf[8..16].try_into().unwrap()),
+            committed_batch_id: u64::from_le_bytes(buf[16..24].try_into().unwrap()),
+            coord: StreamCoordinate::from_bytes(coord_bytes),
+            dedup_count: u32::from_le_bytes(buf[56..60].try_into().unwrap()) as u64,
+        })
+    }
+}
+
+pub struct DualMetadataManager {
+    meta_file: File,
+    dedup_file: File,
+    epoch: u64,
+    committed_batch_id: u64,
+    committed_coord: StreamCoordinate,
+    in_flight_coord: StreamCoordinate,
+    staged_coords: HashMap<u64, StreamCoordinate>,
+    pub deduplicator: TwoStageDeduplicator,
+}
+
+impl DualMetadataManager {
+    pub fn open(meta_path: impl AsRef<Path>) -> Result<Self> {
+        let meta_path = meta_path.as_ref();
+        if let Some(parent) = meta_path.parent()
+            && !parent.as_os_str().is_empty()
+        {
+            std::fs::create_dir_all(parent)?;
+        }
+
+        let dedup_path = Self::dedup_path_for(meta_path);
+
+        let mut meta_file = OpenOptions::new()
+            .read(true)
+            .write(true)
+            .create(true)
+            .truncate(false)
+            .open(meta_path)?;
+
+        let mut dedup_file = OpenOptions::new()
+            .read(true)
+            .write(true)
+            .create(true)
+            .truncate(false)
+            .open(&dedup_path)?;
+
+        let file_len = meta_file.metadata()?.len();
+        let active_slot = if file_len < TOTAL_META_SIZE {
+            meta_file.set_len(TOTAL_META_SIZE)?;
+            let initial = DiskSlot::default();
+            let encoded = initial.encode();
+            meta_file.seek(SeekFrom::Start(0))?;
+            meta_file.write_all(&encoded)?;
+            meta_file.write_all(&encoded)?;
+            meta_file.sync_data()?;
+            initial
+        } else {
+            let mut raw = [0u8; SLOT_SIZE * 2];
+            meta_file.seek(SeekFrom::Start(0))?;
+            meta_file.read_exact(&mut raw)?;
+
+            let slot_a = DiskSlot::decode(raw[0..SLOT_SIZE].try_into().unwrap());
+            let slot_b = DiskSlot::decode(raw[SLOT_SIZE..SLOT_SIZE * 2].try_into().unwrap());
+
+            match (slot_a, slot_b) {
+                (Some(a), Some(b)) => {
+                    if a.epoch >= b.epoch {
+                        a
+                    } else {
+                        b
+                    }
+                }
+                (Some(a), None) => a,
+                (None, Some(b)) => b,
+                (None, None) => return Err(DataLoaderError::CorruptedMetadata),
+            }
+        };
+
+        let valid_dedup_bytes = active_slot.dedup_count * 8;
+        if dedup_file.metadata()?.len() < valid_dedup_bytes {
+            return Err(DataLoaderError::CorruptedMetadata);
+        }
+        dedup_file.set_len(valid_dedup_bytes)?;
+        dedup_file.seek(SeekFrom::Start(0))?;
+
+        let mut raw_hashes = vec![0u8; valid_dedup_bytes as usize];
+        if valid_dedup_bytes > 0 {
+            dedup_file.read_exact(&mut raw_hashes)?;
+        }
+
+        let restored_hashes = raw_hashes
+            .as_chunks::<8>()
+            .0
+            .iter()
+            .map(|&c| u64::from_le_bytes(c));
+
+        let deduplicator = TwoStageDeduplicator::with_committed(restored_hashes);
+
+        Ok(Self {
+            meta_file,
+            dedup_file,
+            epoch: active_slot.epoch,
+            committed_batch_id: active_slot.committed_batch_id,
+            committed_coord: active_slot.coord,
+            in_flight_coord: active_slot.coord,
+            staged_coords: HashMap::new(),
+            deduplicator,
+        })
+    }
+
+    fn dedup_path_for(meta_path: &Path) -> PathBuf {
+        let mut p = meta_path.as_os_str().to_owned();
+        p.push(DEDUP_FILE_SUFFIX);
+        PathBuf::from(p)
+    }
+
+    #[inline]
+    pub fn committed_coordinate(&self) -> StreamCoordinate {
+        self.committed_coord
+    }
+
+    #[inline]
+    pub fn in_flight_coordinate(&self) -> StreamCoordinate {
+        self.in_flight_coord
+    }
+
+    #[inline]
+    pub fn committed_batch_id(&self) -> u64 {
+        self.committed_batch_id
+    }
+
+    #[inline]
+    pub fn set_in_flight_coordinate(&mut self, next_coord: StreamCoordinate) {
+        self.in_flight_coord = next_coord;
+    }
+
+    pub fn stage_batch_in_ram(&mut self, batch_id: u64, next_coord: StreamCoordinate) {
+        self.in_flight_coord = next_coord;
+        self.staged_coords.insert(batch_id, next_coord);
+    }
+
+    pub fn commit_gpu_batch(&mut self, batch_id: u64) -> Result<StreamCoordinate> {
+        let Some(next_coord) = self.staged_coords.remove(&batch_id) else {
+            return Err(DataLoaderError::UnknownBatchId(batch_id));
+        };
+
+        let prev_dedup_count = self.deduplicator.committed_len() as u64;
+        let new_hashes = self.deduplicator.commit_batch(batch_id);
+        if !new_hashes.is_empty() {
+            let mut hash_buf = Vec::with_capacity(new_hashes.len() * 8);
+            for h in new_hashes {
+                hash_buf.extend_from_slice(&h.to_le_bytes());
+            }
+            let write_offset = prev_dedup_count * 8;
+            self.dedup_file.seek(SeekFrom::Start(write_offset))?;
+            self.dedup_file.write_all(&hash_buf)?;
+            self.dedup_file
+                .set_len(write_offset + hash_buf.len() as u64)?;
+            self.dedup_file.sync_data()?;
+        }
+
+        self.epoch += 1;
+        self.committed_batch_id = batch_id;
+        self.committed_coord = next_coord;
+
+        let slot = DiskSlot {
+            epoch: self.epoch,
+            committed_batch_id: self.committed_batch_id,
+            coord: self.committed_coord,
+            dedup_count: self.deduplicator.committed_len() as u64,
+        };
+
+        let slot_offset = if self.epoch % 2 == 1 {
+            0
+        } else {
+            SLOT_SIZE as u64
+        };
+
+        let encoded = slot.encode();
+        self.meta_file.seek(SeekFrom::Start(slot_offset))?;
+        self.meta_file.write_all(&encoded)?;
+        self.meta_file.sync_data()?;
+
+        Ok(self.committed_coord)
+    }
+
+    pub fn rollback_staged_batch(&mut self, batch_id: u64) -> Result<StreamCoordinate> {
+        if self.staged_coords.remove(&batch_id).is_none() {
+            return Err(DataLoaderError::UnknownBatchId(batch_id));
+        }
+        self.deduplicator.rollback_batch(batch_id);
+        self.in_flight_coord = self
+            .staged_coords
+            .iter()
+            .max_by_key(|&(&id, _)| id)
+            .map(|(_, &coord)| coord)
+            .unwrap_or(self.committed_coord);
+        Ok(self.in_flight_coord)
+    }
+
+    pub fn rollback_ram_to_committed(&mut self) -> StreamCoordinate {
+        self.staged_coords.clear();
+        self.deduplicator.clear_staged();
+        self.in_flight_coord = self.committed_coord;
+        self.committed_coord
+    }
+}

data/src/minhash.rs

@@ -1,186 +1,186 @@
-use hashbrown::{HashMap, HashSet};
-use xxhash_rust::xxh3::{xxh3_64, xxh3_64_with_seed};
-
-pub const DEFAULT_MINHASH_PERMUTATIONS: usize = 64;
-
-#[derive(Debug, Clone)]
-pub struct MinHashSignature {
-    pub values: Vec<u64>,
-}
-
-impl MinHashSignature {
-    pub fn compute_char_shingles(text: &str, k: usize, num_perm: usize) -> Self {
-        let mut mins = vec![u64::MAX; num_perm];
-        if text.is_empty() {
-            return Self { values: mins };
-        }
-
-        if text.is_ascii() {
-            let raw = text.as_bytes();
-            let effective_k = k.min(raw.len()).max(1);
-            for bytes in raw.windows(effective_k) {
-                for (i, min_val) in mins.iter_mut().enumerate() {
-                    let seed = (i as u64).wrapping_mul(0x517cc1b727220a95);
-                    let h = xxh3_64_with_seed(bytes, seed);
-                    if h < *min_val {
-                        *min_val = h;
-                    }
-                }
-            }
-            return Self { values: mins };
-        }
-
-        let char_offsets: Vec<usize> = text.char_indices().map(|(idx, _)| idx).collect();
-        let num_chars = char_offsets.len();
-        let effective_k = k.min(num_chars).max(1);
-
-        for start_idx in 0..=(num_chars - effective_k) {
-            let byte_start = char_offsets[start_idx];
-            let byte_end = if start_idx + effective_k < num_chars {
-                char_offsets[start_idx + effective_k]
-            } else {
-                text.len()
-            };
-            let bytes = &text.as_bytes()[byte_start..byte_end];
-
-            for (i, min_val) in mins.iter_mut().enumerate() {
-                let seed = (i as u64).wrapping_mul(0x517cc1b727220a95);
-                let h = xxh3_64_with_seed(bytes, seed);
-                if h < *min_val {
-                    *min_val = h;
-                }
-            }
-        }
-
-        Self { values: mins }
-    }
-
-    pub fn jaccard_similarity(&self, other: &Self) -> f64 {
-        if self.values.is_empty() || self.values.len() != other.values.len() {
-            return 0.0;
-        }
-
-        let mut matches = 0usize;
-        for (a, b) in self.values.iter().zip(other.values.iter()) {
-            if a == b {
-                matches += 1;
-            }
-        }
-
-        (matches as f64) / (self.values.len() as f64)
-    }
-}
-
-pub struct MinHashDeduplicator {
-    threshold: f64,
-    k_shingle: usize,
-    num_perm: usize,
-    num_bands: usize,
-    rows_per_band: usize,
-    signatures: Vec<MinHashSignature>,
-    lsh_buckets: HashMap<(usize, u64), Vec<usize>>,
-}
-
-impl MinHashDeduplicator {
-    pub fn new(threshold: f64, k_shingle: usize, num_perm: usize) -> Self {
-        let n_perm = num_perm.max(8);
-        let num_bands = if n_perm >= 64 { 8 } else { 4.min(n_perm) };
-        let rows_per_band = n_perm / num_bands;
-
-        Self {
-            threshold: threshold.clamp(0.0, 1.0),
-            k_shingle: k_shingle.max(1),
-            num_perm: n_perm,
-            num_bands,
-            rows_per_band,
-            signatures: Vec::new(),
-            lsh_buckets: HashMap::new(),
-        }
-    }
-
-    pub fn compute_signature(&self, text: &str) -> MinHashSignature {
-        MinHashSignature::compute_char_shingles(text, self.k_shingle, self.num_perm)
-    }
-
-    fn band_hash(&self, sig: &MinHashSignature, band_idx: usize) -> u64 {
-        let start = band_idx * self.rows_per_band;
-        let end = (start + self.rows_per_band).min(sig.values.len());
-        let slice = &sig.values[start..end];
-        let mut byte_buf = Vec::with_capacity(slice.len() * 8);
-        for &v in slice {
-            byte_buf.extend_from_slice(&v.to_le_bytes());
-        }
-        xxh3_64(&byte_buf)
-    }
-
-    pub fn max_similarity(&self, sig: &MinHashSignature) -> f64 {
-        if self.signatures.is_empty() {
-            return 0.0;
-        }
-
-        // For datasets under 128 items, perform exact scan for 100% precision
-        if self.signatures.len() <= 128 {
-            let mut max_sim = 0.0f64;
-            for existing in &self.signatures {
-                let sim = sig.jaccard_similarity(existing);
-                if sim > max_sim {
-                    max_sim = sim;
-                    if max_sim >= self.threshold {
-                        return max_sim;
-                    }
-                }
-            }
-            return max_sim;
-        }
-
-        // For larger datasets, query LSH candidate buckets in O(1) amortized time
-        let mut candidates = HashSet::new();
-        for b in 0..self.num_bands {
-            let bh = self.band_hash(sig, b);
-            if let Some(doc_ids) = self.lsh_buckets.get(&(b, bh)) {
-                for &id in doc_ids {
-                    candidates.insert(id);
-                }
-            }
-        }
-
-        let mut max_sim = 0.0f64;
-        for &cand_id in &candidates {
-            if let Some(existing) = self.signatures.get(cand_id) {
-                let sim = sig.jaccard_similarity(existing);
-                if sim > max_sim {
-                    max_sim = sim;
-                    if max_sim >= self.threshold {
-                        return max_sim;
-                    }
-                }
-            }
-        }
-        max_sim
-    }
-
-    pub fn is_unique(&mut self, text: &str) -> bool {
-        let sig = self.compute_signature(text);
-        let max_sim = self.max_similarity(&sig);
-        if max_sim >= self.threshold {
-            return false;
-        }
-
-        let doc_id = self.signatures.len();
-        for b in 0..self.num_bands {
-            let bh = self.band_hash(&sig, b);
-            self.lsh_buckets.entry((b, bh)).or_default().push(doc_id);
-        }
-        self.signatures.push(sig);
-        true
-    }
-
-    pub fn count(&self) -> usize {
-        self.signatures.len()
-    }
-
-    pub fn clear(&mut self) {
-        self.signatures.clear();
-        self.lsh_buckets.clear();
-    }
-}
+use hashbrown::{HashMap, HashSet};
+use xxhash_rust::xxh3::{xxh3_64, xxh3_64_with_seed};
+
+pub const DEFAULT_MINHASH_PERMUTATIONS: usize = 64;
+
+#[derive(Debug, Clone)]
+pub struct MinHashSignature {
+    pub values: Vec<u64>,
+}
+
+impl MinHashSignature {
+    pub fn compute_char_shingles(text: &str, k: usize, num_perm: usize) -> Self {
+        let mut mins = vec![u64::MAX; num_perm];
+        if text.is_empty() {
+            return Self { values: mins };
+        }
+
+        if text.is_ascii() {
+            let raw = text.as_bytes();
+            let effective_k = k.min(raw.len()).max(1);
+            for bytes in raw.windows(effective_k) {
+                for (i, min_val) in mins.iter_mut().enumerate() {
+                    let seed = (i as u64).wrapping_mul(0x517cc1b727220a95);
+                    let h = xxh3_64_with_seed(bytes, seed);
+                    if h < *min_val {
+                        *min_val = h;
+                    }
+                }
+            }
+            return Self { values: mins };
+        }
+
+        let char_offsets: Vec<usize> = text.char_indices().map(|(idx, _)| idx).collect();
+        let num_chars = char_offsets.len();
+        let effective_k = k.min(num_chars).max(1);
+
+        for start_idx in 0..=(num_chars - effective_k) {
+            let byte_start = char_offsets[start_idx];
+            let byte_end = if start_idx + effective_k < num_chars {
+                char_offsets[start_idx + effective_k]
+            } else {
+                text.len()
+            };
+            let bytes = &text.as_bytes()[byte_start..byte_end];
+
+            for (i, min_val) in mins.iter_mut().enumerate() {
+                let seed = (i as u64).wrapping_mul(0x517cc1b727220a95);
+                let h = xxh3_64_with_seed(bytes, seed);
+                if h < *min_val {
+                    *min_val = h;
+                }
+            }
+        }
+
+        Self { values: mins }
+    }
+
+    pub fn jaccard_similarity(&self, other: &Self) -> f64 {
+        if self.values.is_empty() || self.values.len() != other.values.len() {
+            return 0.0;
+        }
+
+        let mut matches = 0usize;
+        for (a, b) in self.values.iter().zip(other.values.iter()) {
+            if a == b {
+                matches += 1;
+            }
+        }
+
+        (matches as f64) / (self.values.len() as f64)
+    }
+}
+
+pub struct MinHashDeduplicator {
+    threshold: f64,
+    k_shingle: usize,
+    num_perm: usize,
+    num_bands: usize,
+    rows_per_band: usize,
+    signatures: Vec<MinHashSignature>,
+    lsh_buckets: HashMap<(usize, u64), Vec<usize>>,
+}
+
+impl MinHashDeduplicator {
+    pub fn new(threshold: f64, k_shingle: usize, num_perm: usize) -> Self {
+        let n_perm = num_perm.max(8);
+        let num_bands = if n_perm >= 64 { 8 } else { 4.min(n_perm) };
+        let rows_per_band = n_perm / num_bands;
+
+        Self {
+            threshold: threshold.clamp(0.0, 1.0),
+            k_shingle: k_shingle.max(1),
+            num_perm: n_perm,
+            num_bands,
+            rows_per_band,
+            signatures: Vec::new(),
+            lsh_buckets: HashMap::new(),
+        }
+    }
+
+    pub fn compute_signature(&self, text: &str) -> MinHashSignature {
+        MinHashSignature::compute_char_shingles(text, self.k_shingle, self.num_perm)
+    }
+
+    fn band_hash(&self, sig: &MinHashSignature, band_idx: usize) -> u64 {
+        let start = band_idx * self.rows_per_band;
+        let end = (start + self.rows_per_band).min(sig.values.len());
+        let slice = &sig.values[start..end];
+        let mut byte_buf = Vec::with_capacity(slice.len() * 8);
+        for &v in slice {
+            byte_buf.extend_from_slice(&v.to_le_bytes());
+        }
+        xxh3_64(&byte_buf)
+    }
+
+    pub fn max_similarity(&self, sig: &MinHashSignature) -> f64 {
+        if self.signatures.is_empty() {
+            return 0.0;
+        }
+
+        // For datasets under 128 items, perform exact scan for 100% precision
+        if self.signatures.len() <= 128 {
+            let mut max_sim = 0.0f64;
+            for existing in &self.signatures {
+                let sim = sig.jaccard_similarity(existing);
+                if sim > max_sim {
+                    max_sim = sim;
+                    if max_sim >= self.threshold {
+                        return max_sim;
+                    }
+                }
+            }
+            return max_sim;
+        }
+
+        // For larger datasets, query LSH candidate buckets in O(1) amortized time
+        let mut candidates = HashSet::new();
+        for b in 0..self.num_bands {
+            let bh = self.band_hash(sig, b);
+            if let Some(doc_ids) = self.lsh_buckets.get(&(b, bh)) {
+                for &id in doc_ids {
+                    candidates.insert(id);
+                }
+            }
+        }
+
+        let mut max_sim = 0.0f64;
+        for &cand_id in &candidates {
+            if let Some(existing) = self.signatures.get(cand_id) {
+                let sim = sig.jaccard_similarity(existing);
+                if sim > max_sim {
+                    max_sim = sim;
+                    if max_sim >= self.threshold {
+                        return max_sim;
+                    }
+                }
+            }
+        }
+        max_sim
+    }
+
+    pub fn is_unique(&mut self, text: &str) -> bool {
+        let sig = self.compute_signature(text);
+        let max_sim = self.max_similarity(&sig);
+        if max_sim >= self.threshold {
+            return false;
+        }
+
+        let doc_id = self.signatures.len();
+        for b in 0..self.num_bands {
+            let bh = self.band_hash(&sig, b);
+            self.lsh_buckets.entry((b, bh)).or_default().push(doc_id);
+        }
+        self.signatures.push(sig);
+        true
+    }
+
+    pub fn count(&self) -> usize {
+        self.signatures.len()
+    }
+
+    pub fn clear(&mut self) {
+        self.signatures.clear();
+        self.lsh_buckets.clear();
+    }
+}

data/src/postprocess.rs

@@ -1,135 +1,135 @@
-use rayon::prelude::*;
-use serde_json::Value;
-
-#[derive(Debug, Clone)]
-pub struct PostProcessConfig {
-    pub trim_whitespace: bool,
-    pub normalize_newlines: bool,
-    pub min_byte_len: usize,
-    pub max_byte_len: usize,
-    pub truncate_over_max: bool,
-    pub json_extract_key: Option<String>,
-}
-
-impl Default for PostProcessConfig {
-    fn default() -> Self {
-        Self {
-            trim_whitespace: true,
-            normalize_newlines: true,
-            min_byte_len: 0,
-            max_byte_len: 0,
-            truncate_over_max: false,
-            json_extract_key: None,
-        }
-    }
-}
-
-pub fn extract_json_value_by_path<'a>(root: &'a Value, key_path: &str) -> Option<&'a Value> {
-    if let Some(direct) = root.get(key_path) {
-        return Some(direct);
-    }
-    if !key_path.contains('.') {
-        return None;
-    }
-    let mut current = root;
-    for segment in key_path.split('.') {
-        if segment.is_empty() {
-            continue;
-        }
-        match current {
-            Value::Object(map) => {
-                current = map.get(segment)?;
-            }
-            Value::Array(arr) => {
-                let idx: isize = segment.parse().ok()?;
-                let resolved = if idx < 0 {
-                    let abs = idx.unsigned_abs();
-                    if abs > arr.len() {
-                        return None;
-                    }
-                    arr.len() - abs
-                } else {
-                    idx as usize
-                };
-                current = arr.get(resolved)?;
-            }
-            _ => return None,
-        }
-    }
-    Some(current)
-}
-
-fn json_value_to_extracted_string(val: &Value) -> Option<String> {
-    match val {
-        Value::Null => None,
-        Value::String(s) => Some(s.clone()),
-        Value::Bool(b) => Some(b.to_string()),
-        Value::Number(n) => Some(n.to_string()),
-        Value::Array(_) | Value::Object(_) => Some(val.to_string()),
-    }
-}
-
-pub fn collect_json_object_keys(raw_line: &str) -> Option<Vec<String>> {
-    let parsed: Value = serde_json::from_str(raw_line).ok()?;
-    let obj = parsed.as_object()?;
-    Some(obj.keys().cloned().collect())
-}
-
-pub fn process_single_record(raw: &str, cfg: &PostProcessConfig) -> Option<String> {
-    let mut text = if let Some(key) = &cfg.json_extract_key {
-        let parsed: Value = serde_json::from_str(raw).ok()?;
-        let val = extract_json_value_by_path(&parsed, key)?;
-        json_value_to_extracted_string(val)?
-    } else {
-        raw.to_owned()
-    };
-
-    if cfg.normalize_newlines && crate::simd_cpu::contains_byte_fast(text.as_bytes(), b'\r') {
-        text = text.replace("\r\n", "\n").replace('\r', "\n");
-    }
-
-    if cfg.trim_whitespace {
-        let trimmed = text.trim();
-        if trimmed.len() != text.len() {
-            text = trimmed.to_owned();
-        }
-    }
-
-    if cfg.min_byte_len > 0 && text.len() < cfg.min_byte_len {
-        return None;
-    }
-
-    if cfg.max_byte_len > 0 && text.len() > cfg.max_byte_len {
-        if cfg.truncate_over_max {
-            let mut cut = cfg.max_byte_len;
-            while cut > 0 && !text.is_char_boundary(cut) {
-                cut -= 1;
-            }
-            text.truncate(cut);
-            if cfg.min_byte_len > 0 && text.len() < cfg.min_byte_len {
-                return None;
-            }
-        } else {
-            return None;
-        }
-    }
-
-    Some(text)
-}
-
-pub fn process_batch_parallel(raw_records: &[String], cfg: &PostProcessConfig) -> Vec<String> {
-    raw_records
-        .par_iter()
-        .filter_map(|item| process_single_record(item, cfg))
-        .collect()
-}
-
-pub fn process_batch_parallel_indexed(
-    raw_records: &[String],
-    cfg: &PostProcessConfig,
-) -> Vec<Option<String>> {
-    raw_records
-        .par_iter()
-        .map(|item| process_single_record(item, cfg))
-        .collect()
-}
+use rayon::prelude::*;
+use serde_json::Value;
+
+#[derive(Debug, Clone)]
+pub struct PostProcessConfig {
+    pub trim_whitespace: bool,
+    pub normalize_newlines: bool,
+    pub min_byte_len: usize,
+    pub max_byte_len: usize,
+    pub truncate_over_max: bool,
+    pub json_extract_key: Option<String>,
+}
+
+impl Default for PostProcessConfig {
+    fn default() -> Self {
+        Self {
+            trim_whitespace: true,
+            normalize_newlines: true,
+            min_byte_len: 0,
+            max_byte_len: 0,
+            truncate_over_max: false,
+            json_extract_key: None,
+        }
+    }
+}
+
+pub fn extract_json_value_by_path<'a>(root: &'a Value, key_path: &str) -> Option<&'a Value> {
+    if let Some(direct) = root.get(key_path) {
+        return Some(direct);
+    }
+    if !key_path.contains('.') {
+        return None;
+    }
+    let mut current = root;
+    for segment in key_path.split('.') {
+        if segment.is_empty() {
+            continue;
+        }
+        match current {
+            Value::Object(map) => {
+                current = map.get(segment)?;
+            }
+            Value::Array(arr) => {
+                let idx: isize = segment.parse().ok()?;
+                let resolved = if idx < 0 {
+                    let abs = idx.unsigned_abs();
+                    if abs > arr.len() {
+                        return None;
+                    }
+                    arr.len() - abs
+                } else {
+                    idx as usize
+                };
+                current = arr.get(resolved)?;
+            }
+            _ => return None,
+        }
+    }
+    Some(current)
+}
+
+fn json_value_to_extracted_string(val: &Value) -> Option<String> {
+    match val {
+        Value::Null => None,
+        Value::String(s) => Some(s.clone()),
+        Value::Bool(b) => Some(b.to_string()),
+        Value::Number(n) => Some(n.to_string()),
+        Value::Array(_) | Value::Object(_) => Some(val.to_string()),
+    }
+}
+
+pub fn collect_json_object_keys(raw_line: &str) -> Option<Vec<String>> {
+    let parsed: Value = serde_json::from_str(raw_line).ok()?;
+    let obj = parsed.as_object()?;
+    Some(obj.keys().cloned().collect())
+}
+
+pub fn process_single_record(raw: &str, cfg: &PostProcessConfig) -> Option<String> {
+    let mut text = if let Some(key) = &cfg.json_extract_key {
+        let parsed: Value = serde_json::from_str(raw).ok()?;
+        let val = extract_json_value_by_path(&parsed, key)?;
+        json_value_to_extracted_string(val)?
+    } else {
+        raw.to_owned()
+    };
+
+    if cfg.normalize_newlines && crate::simd_cpu::contains_byte_fast(text.as_bytes(), b'\r') {
+        text = text.replace("\r\n", "\n").replace('\r', "\n");
+    }
+
+    if cfg.trim_whitespace {
+        let trimmed = text.trim();
+        if trimmed.len() != text.len() {
+            text = trimmed.to_owned();
+        }
+    }
+
+    if cfg.min_byte_len > 0 && text.len() < cfg.min_byte_len {
+        return None;
+    }
+
+    if cfg.max_byte_len > 0 && text.len() > cfg.max_byte_len {
+        if cfg.truncate_over_max {
+            let mut cut = cfg.max_byte_len;
+            while cut > 0 && !text.is_char_boundary(cut) {
+                cut -= 1;
+            }
+            text.truncate(cut);
+            if cfg.min_byte_len > 0 && text.len() < cfg.min_byte_len {
+                return None;
+            }
+        } else {
+            return None;
+        }
+    }
+
+    Some(text)
+}
+
+pub fn process_batch_parallel(raw_records: &[String], cfg: &PostProcessConfig) -> Vec<String> {
+    raw_records
+        .par_iter()
+        .filter_map(|item| process_single_record(item, cfg))
+        .collect()
+}
+
+pub fn process_batch_parallel_indexed(
+    raw_records: &[String],
+    cfg: &PostProcessConfig,
+) -> Vec<Option<String>> {
+    raw_records
+        .par_iter()
+        .map(|item| process_single_record(item, cfg))
+        .collect()
+}

data/src/py_bridge.rs

@@ -1,839 +1,839 @@
-use std::fs::File;
-use std::io::{Read, Write};
-
-use parking_lot::Mutex;
-use pyo3::exceptions::{PyRuntimeError, PyValueError};
-use pyo3::prelude::*;
-use pyo3::types::{PyBool, PyByteArray, PyBytes, PyDict, PyFloat, PyInt, PySequence, PyString};
-use rayon::prelude::*;
-
-use crate::coord::StreamCoordinate;
-use crate::dedup::TwoStageDeduplicator;
-use crate::error::{DataLoaderError, find_closest_matches};
-use crate::metadata::DualMetadataManager;
-use crate::postprocess::PostProcessConfig;
-use crate::quality::QualityConfig;
-use crate::serialize::{DataValue, classify_string_or_json, serialize_columns_by_format};
-
-fn to_py_err(err: DataLoaderError) -> PyErr {
-    match err {
-        DataLoaderError::InvalidConfig(msg) | DataLoaderError::ColumnNotFound(msg) => {
-            PyValueError::new_err(msg)
-        }
-        DataLoaderError::InvalidCoordinate => PyValueError::new_err(err.to_string()),
-        other => PyRuntimeError::new_err(other.to_string()),
-    }
-}
-
-#[pyclass(frozen, name = "StreamCoordinate")]
-#[derive(Clone, Copy)]
-pub struct PyStreamCoordinate {
-    #[pyo3(get)]
-    pub shard_id: u32,
-    #[pyo3(get)]
-    pub byte_offset: u64,
-    #[pyo3(get)]
-    pub row_group_idx: u32,
-    #[pyo3(get)]
-    pub page_idx: u32,
-    #[pyo3(get)]
-    pub row_offset_in_page: u32,
-    #[pyo3(get)]
-    pub global_row_idx: u64,
-}
-
-#[pymethods]
-impl PyStreamCoordinate {
-    #[new]
-    #[pyo3(signature = (
-        shard_id = 0,
-        byte_offset = 0,
-        row_group_idx = 0,
-        page_idx = 0,
-        row_offset_in_page = 0,
-        global_row_idx = 0,
-    ))]
-    pub fn new(
-        shard_id: u32,
-        byte_offset: u64,
-        row_group_idx: u32,
-        page_idx: u32,
-        row_offset_in_page: u32,
-        global_row_idx: u64,
-    ) -> Self {
-        Self {
-            shard_id,
-            byte_offset,
-            row_group_idx,
-            page_idx,
-            row_offset_in_page,
-            global_row_idx,
-        }
-    }
-
-    pub fn to_bytes<'py>(&self, py: Python<'py>) -> Bound<'py, PyBytes> {
-        let coord = StreamCoordinate {
-            shard_id: self.shard_id,
-            byte_offset: self.byte_offset,
-            row_group_idx: self.row_group_idx,
-            page_idx: self.page_idx,
-            row_offset_in_page: self.row_offset_in_page,
-            global_row_idx: self.global_row_idx,
-        };
-        PyBytes::new(py, &coord.to_bytes())
-    }
-
-    #[staticmethod]
-    pub fn from_bytes(raw: &[u8]) -> PyResult<Self> {
-        if raw.len() != StreamCoordinate::BYTE_LEN {
-            return Err(PyValueError::new_err(format!(
-                "Invalid coordinate length: expected {} bytes, got {}",
-                StreamCoordinate::BYTE_LEN,
-                raw.len()
-            )));
-        }
-        let arr: &[u8; StreamCoordinate::BYTE_LEN] = raw.try_into().unwrap();
-        let c = StreamCoordinate::from_bytes(arr);
-        Ok(Self {
-            shard_id: c.shard_id,
-            byte_offset: c.byte_offset,
-            row_group_idx: c.row_group_idx,
-            page_idx: c.page_idx,
-            row_offset_in_page: c.row_offset_in_page,
-            global_row_idx: c.global_row_idx,
-        })
-    }
-
-    pub fn __repr__(&self) -> String {
-        format!(
-            "StreamCoordinate(shard_id={}, byte_offset={}, row_group={}, page={}, row_offset={}, global_row={})",
-            self.shard_id,
-            self.byte_offset,
-            self.row_group_idx,
-            self.page_idx,
-            self.row_offset_in_page,
-            self.global_row_idx
-        )
-    }
-}
-
-#[pyclass(name = "StreamDeduplicator")]
-pub struct PyStreamDeduplicator {
-    inner: Mutex<TwoStageDeduplicator>,
-}
-
-#[pymethods]
-impl PyStreamDeduplicator {
-    #[new]
-    #[pyo3(signature = (initial_hashes = None))]
-    pub fn new(initial_hashes: Option<Vec<u64>>) -> Self {
-        let dedup = match initial_hashes {
-            Some(hashes) => TwoStageDeduplicator::with_committed(hashes),
-            None => TwoStageDeduplicator::new(),
-        };
-        Self {
-            inner: Mutex::new(dedup),
-        }
-    }
-
-    #[staticmethod]
-    pub fn compute_hash(payload: &[u8]) -> u64 {
-        TwoStageDeduplicator::compute_hash(payload)
-    }
-
-    pub fn is_seen(&self, payload: &[u8]) -> bool {
-        self.inner.lock().is_seen(payload)
-    }
-
-    pub fn try_add(&self, payload: &[u8]) -> bool {
-        let mut dedup = self.inner.lock();
-        dedup.try_stage(1, payload).is_some()
-    }
-
-    pub fn filter_batch(&self, records: Vec<String>) -> Vec<String> {
-        let mut dedup = self.inner.lock();
-        let mut out = Vec::with_capacity(records.len());
-        for rec in records {
-            if dedup.try_stage(1, rec.as_bytes()).is_some() {
-                out.push(rec);
-            }
-        }
-        out
-    }
-
-    pub fn commit(&self) -> usize {
-        let mut dedup = self.inner.lock();
-        dedup.commit_batch(1).len()
-    }
-
-    pub fn rollback(&self) -> usize {
-        let mut dedup = self.inner.lock();
-        dedup.rollback_batch(1)
-    }
-
-    pub fn clear(&self) {
-        let mut dedup = self.inner.lock();
-        dedup.clear_staged();
-    }
-
-    pub fn count(&self) -> usize {
-        self.inner.lock().committed_len()
-    }
-
-    pub fn save_wal(&self, path: &str) -> PyResult<usize> {
-        let dedup = self.inner.lock();
-        let hashes = dedup.export_committed_hashes();
-        let mut buf = Vec::with_capacity(hashes.len() * 8);
-        for h in &hashes {
-            buf.extend_from_slice(&h.to_le_bytes());
-        }
-        let mut file = File::create(path).map_err(|e| PyRuntimeError::new_err(e.to_string()))?;
-        file.write_all(&buf)
-            .map_err(|e| PyRuntimeError::new_err(e.to_string()))?;
-        file.sync_data()
-            .map_err(|e| PyRuntimeError::new_err(e.to_string()))?;
-        Ok(hashes.len())
-    }
-
-    pub fn load_wal(&self, path: &str) -> PyResult<usize> {
-        let mut file = match File::open(path) {
-            Ok(f) => f,
-            Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(0),
-            Err(e) => return Err(PyRuntimeError::new_err(e.to_string())),
-        };
-        let mut buf = Vec::new();
-        file.read_to_end(&mut buf)
-            .map_err(|e| PyRuntimeError::new_err(e.to_string()))?;
-        if buf.len() % 8 != 0 {
-            return Err(PyValueError::new_err(format!(
-                "Corrupted WAL file '{}': byte length {} is not a multiple of 8",
-                path,
-                buf.len()
-            )));
-        }
-        let count = buf.len() / 8;
-        let mut dedup = self.inner.lock();
-        let (chunks, _) = buf.as_chunks::<8>();
-        for chunk in chunks {
-            let h = u64::from_le_bytes(*chunk);
-            dedup.import_committed_hashes(std::iter::once(h));
-        }
-        Ok(count)
-    }
-
-    pub fn export_hashes(&self) -> Vec<u64> {
-        self.inner.lock().export_committed_hashes()
-    }
-}
-
-#[pyclass(name = "StateTracker")]
-pub struct PyStateTracker {
-    inner: Mutex<DualMetadataManager>,
-}
-
-#[pymethods]
-impl PyStateTracker {
-    #[new]
-    pub fn new(meta_path: &str) -> PyResult<Self> {
-        let mgr = DualMetadataManager::open(meta_path).map_err(to_py_err)?;
-        Ok(Self {
-            inner: Mutex::new(mgr),
-        })
-    }
-
-    pub fn read(&self) -> PyStreamCoordinate {
-        let mgr = self.inner.lock();
-        let c = mgr.committed_coordinate();
-        PyStreamCoordinate {
-            shard_id: c.shard_id,
-            byte_offset: c.byte_offset,
-            row_group_idx: c.row_group_idx,
-            page_idx: c.page_idx,
-            row_offset_in_page: c.row_offset_in_page,
-            global_row_idx: c.global_row_idx,
-        }
-    }
-
-    #[pyo3(signature = (
-        shard_id = 0,
-        byte_offset = 0,
-        row_group_idx = 0,
-        page_idx = 0,
-        row_offset_in_page = 0,
-        global_row_idx = 0,
-        batch_id = 1
-    ))]
-    #[allow(clippy::too_many_arguments)]
-    pub fn update(
-        &self,
-        shard_id: u32,
-        byte_offset: u64,
-        row_group_idx: u32,
-        page_idx: u32,
-        row_offset_in_page: u32,
-        global_row_idx: u64,
-        batch_id: u64,
-    ) {
-        let coord = StreamCoordinate {
-            shard_id,
-            byte_offset,
-            row_group_idx,
-            page_idx,
-            row_offset_in_page,
-            global_row_idx,
-        };
-        let mut mgr = self.inner.lock();
-        mgr.stage_batch_in_ram(batch_id, coord);
-    }
-
-    #[pyo3(signature = (batch_id = 1))]
-    pub fn commit(&self, batch_id: u64) -> PyResult<PyStreamCoordinate> {
-        let mut mgr = self.inner.lock();
-        let c = mgr.commit_gpu_batch(batch_id).map_err(to_py_err)?;
-        Ok(PyStreamCoordinate {
-            shard_id: c.shard_id,
-            byte_offset: c.byte_offset,
-            row_group_idx: c.row_group_idx,
-            page_idx: c.page_idx,
-            row_offset_in_page: c.row_offset_in_page,
-            global_row_idx: c.global_row_idx,
-        })
-    }
-
-    pub fn committed_batch_id(&self) -> u64 {
-        self.inner.lock().committed_batch_id()
-    }
-}
-
-#[pyclass(name = "QualityFilter")]
-pub struct PyQualityFilter {
-    cfg: QualityConfig,
-}
-
-#[pymethods]
-impl PyQualityFilter {
-    #[new]
-    #[pyo3(signature = (
-        min_chars = 0,
-        max_chars = 0,
-        min_words = 0,
-        min_entropy = 0.0,
-        max_repetition_ratio = 1.0,
-        max_symbol_ratio = 1.0
-    ))]
-    pub fn new(
-        min_chars: usize,
-        max_chars: usize,
-        min_words: usize,
-        min_entropy: f64,
-        max_repetition_ratio: f64,
-        max_symbol_ratio: f64,
-    ) -> Self {
-        Self {
-            cfg: QualityConfig {
-                min_chars,
-                max_chars,
-                min_words,
-                min_entropy,
-                max_repetition_ratio,
-                max_symbol_ratio,
-            },
-        }
-    }
-
-    pub fn is_high_quality(&self, text: &str) -> bool {
-        crate::quality::evaluate_quality(text, &self.cfg).is_passed
-    }
-
-    pub fn metrics(&self, text: &str) -> (usize, usize, f64, f64, f64, bool) {
-        let m = crate::quality::evaluate_quality(text, &self.cfg);
-        (
-            m.char_count,
-            m.word_count,
-            m.entropy,
-            m.repetition_ratio,
-            m.symbol_ratio,
-            m.is_passed,
-        )
-    }
-
-    pub fn filter_flags(&self, py: Python<'_>, texts: Vec<String>) -> Vec<bool> {
-        let cfg = self.cfg.clone();
-        py.detach(|| crate::quality::filter_quality_batch_parallel(&texts, &cfg))
-    }
-
-    pub fn filter_batch(&self, py: Python<'_>, texts: Vec<String>) -> Vec<String> {
-        let cfg = self.cfg.clone();
-        py.detach(|| {
-            texts
-                .into_iter()
-                .filter(|t| crate::quality::evaluate_quality(t, &cfg).is_passed)
-                .collect()
-        })
-    }
-}
-
-#[pyclass(name = "MinHashDeduplicator")]
-pub struct PyMinHashDeduplicator {
-    inner: Mutex<crate::minhash::MinHashDeduplicator>,
-}
-
-#[pymethods]
-impl PyMinHashDeduplicator {
-    #[new]
-    #[pyo3(signature = (threshold = 0.85, k_shingle = 5, num_perm = 64))]
-    pub fn new(threshold: f64, k_shingle: usize, num_perm: usize) -> Self {
-        Self {
-            inner: Mutex::new(crate::minhash::MinHashDeduplicator::new(
-                threshold, k_shingle, num_perm,
-            )),
-        }
-    }
-
-    pub fn is_unique(&self, text: &str) -> bool {
-        self.inner.lock().is_unique(text)
-    }
-
-    pub fn similarity(&self, text_a: &str, text_b: &str) -> f64 {
-        let dedup = self.inner.lock();
-        let sig_a = dedup.compute_signature(text_a);
-        let sig_b = dedup.compute_signature(text_b);
-        sig_a.jaccard_similarity(&sig_b)
-    }
-
-    pub fn count(&self) -> usize {
-        self.inner.lock().count()
-    }
-
-    pub fn clear(&self) {
-        self.inner.lock().clear();
-    }
-}
-
-#[pyclass(name = "StringTensor")]
-#[derive(Clone)]
-pub struct PyStringTensor {
-    inner: crate::string_tensor::StringTensor,
-}
-
-#[pymethods]
-impl PyStringTensor {
-    #[staticmethod]
-    pub fn from_list(strings: Vec<String>) -> Self {
-        Self {
-            inner: crate::string_tensor::StringTensor::from_strings(strings),
-        }
-    }
-
-    #[staticmethod]
-    pub fn from_gpu_buffers(offsets: Vec<u32>, payload: &[u8]) -> Self {
-        Self {
-            inner: crate::string_tensor::StringTensor::from_gpu_buffers(&offsets, payload),
-        }
-    }
-
-    pub fn to_gpu_buffers<'py>(
-        &self,
-        py: Python<'py>,
-    ) -> (Bound<'py, PyBytes>, Bound<'py, PyBytes>) {
-        let (offsets, payload) = self.inner.to_gpu_buffers();
-        let mut offset_bytes = Vec::with_capacity(offsets.len() * 4);
-        for &off in &offsets {
-            offset_bytes.extend_from_slice(&off.to_le_bytes());
-        }
-        (PyBytes::new(py, &offset_bytes), PyBytes::new(py, &payload))
-    }
-
-    pub fn to_offsets(&self) -> Vec<u32> {
-        self.inner.to_gpu_buffers().0
-    }
-
-    pub fn to_payload<'py>(&self, py: Python<'py>) -> Bound<'py, PyBytes> {
-        let (_, payload) = self.inner.to_gpu_buffers();
-        PyBytes::new(py, &payload)
-    }
-
-    pub fn __len__(&self) -> usize {
-        self.inner.len()
-    }
-
-    pub fn __getitem__(&self, index: usize) -> PyResult<String> {
-        self.inner
-            .get(index)
-            .map(|s| s.to_owned())
-            .ok_or_else(|| PyValueError::new_err("StringTensor index out of range"))
-    }
-
-    pub fn get(&self, index: usize) -> Option<String> {
-        self.inner.get(index).map(|s| s.to_owned())
-    }
-
-    pub fn to_list(&self) -> Vec<String> {
-        self.inner.to_vec()
-    }
-
-    pub fn lengths(&self) -> Vec<u32> {
-        self.inner.lengths()
-    }
-
-    pub fn is_inlined_mask(&self) -> Vec<bool> {
-        self.inner.is_inlined_mask()
-    }
-
-    pub fn total_bytes(&self) -> usize {
-        self.inner.views.iter().map(|v| v.len as usize).sum()
-    }
-
-    pub fn filter_by_length(&self, min_len: u32, max_len: u32) -> Self {
-        Self {
-            inner: self.inner.filter_by_length(min_len, max_len),
-        }
-    }
-
-    pub fn filter_by_prefix(&self, prefix: &str) -> Self {
-        Self {
-            inner: self.inner.filter_by_prefix(prefix),
-        }
-    }
-
-    pub fn hash_all(&self, py: Python<'_>) -> Vec<u64> {
-        py.detach(|| self.inner.hash_all_parallel())
-    }
-
-    pub fn __repr__(&self) -> String {
-        format!(
-            "StringTensor(count={}, inlined={})",
-            self.inner.len(),
-            self.inner.views.iter().filter(|v| v.is_inlined()).count()
-        )
-    }
-}
-
-pub fn extract_py_columns_table(
-    py: Python<'_>,
-    columns: &Bound<'_, PyAny>,
-) -> PyResult<Vec<(String, Vec<DataValue>)>> {
-    let mut rust_cols = Vec::new();
-
-    if let Ok(dict) = columns.cast::<PyDict>() {
-        for (k, v) in dict.iter() {
-            let col_name = k.extract::<String>()?;
-            let vals = extract_py_seq_values(py, &v)?;
-            rust_cols.push((col_name, vals));
-        }
-    } else if let Ok(seq) = columns.cast::<PySequence>() {
-        let len = seq.len()?;
-        for i in 0..len {
-            let item = seq.get_item(i)?;
-            if let Ok(pair) = item.cast::<PySequence>()
-                && pair.len()? == 2
-            {
-                let col_name = pair.get_item(0)?.extract::<String>()?;
-                let col_vals_seq = pair.get_item(1)?;
-                let vals = extract_py_seq_values(py, &col_vals_seq)?;
-                rust_cols.push((col_name, vals));
-                continue;
-            }
-            return Err(PyValueError::new_err(
-                "columns sequence must contain (name, values) pairs or be a dict",
-            ));
-        }
-    } else {
-        return Err(PyValueError::new_err(
-            "columns must be a dict {name: [values]} or sequence of (name, [values]) pairs",
-        ));
-    }
-
-    Ok(rust_cols)
-}
-
-fn extract_py_seq_values(_py: Python<'_>, seq_obj: &Bound<'_, PyAny>) -> PyResult<Vec<DataValue>> {
-    let seq = seq_obj
-        .cast::<PySequence>()
-        .map_err(|_| PyValueError::new_err("column values must be a sequence (list/tuple)"))?;
-    let len = seq.len()?;
-    let mut out = Vec::with_capacity(len);
-
-    for i in 0..len {
-        let item = seq.get_item(i)?;
-        if item.is_none() {
-            out.push(DataValue::Null);
-        } else if let Ok(b) = item.cast::<PyBool>() {
-            out.push(DataValue::Bool(b.is_true()));
-        } else if let Ok(i) = item.cast::<PyInt>() {
-            out.push(DataValue::Int(i.extract::<i64>()?));
-        } else if let Ok(f) = item.cast::<PyFloat>() {
-            out.push(DataValue::Float(f.extract::<f64>()?));
-        } else if let Ok(s) = item.cast::<PyString>() {
-            let str_val = s.to_string_lossy();
-            out.push(classify_string_or_json(&str_val));
-        } else if let Ok(bytes) = item.cast::<PyBytes>() {
-            out.push(DataValue::Bytes(bytes.as_bytes().to_vec()));
-        } else if let Ok(bytes) = item.cast::<PyByteArray>() {
-            out.push(DataValue::Bytes(bytes.to_vec()));
-        } else {
-            let py_str = item.str()?;
-            let str_repr = py_str.to_string_lossy();
-            out.push(classify_string_or_json(&str_repr));
-        }
-    }
-
-    Ok(out)
-}
-
-#[pyfunction]
-pub fn dataloader_cpu_isa() -> &'static str {
-    crate::simd_cpu::detect_cpu_isa().as_str()
-}
-
-#[pyfunction]
-#[pyo3(signature = (target, candidates, max_suggestions=3))]
-pub fn suggest_closest_matches(
-    target: &str,
-    candidates: Vec<String>,
-    max_suggestions: usize,
-) -> Vec<String> {
-    find_closest_matches(target, &candidates, max_suggestions)
-}
-
-#[pyfunction]
-#[pyo3(signature = (columns, format="parquet", hf_image_struct=true))]
-pub fn serialize_table_to_bytes(
-    py: Python<'_>,
-    columns: &Bound<'_, PyAny>,
-    format: &str,
-    hf_image_struct: bool,
-) -> PyResult<Vec<u8>> {
-    let rust_cols = extract_py_columns_table(py, columns)?;
-    py.detach(|| serialize_columns_by_format(&rust_cols, format, hf_image_struct))
-        .map_err(to_py_err)
-}
-
-#[pyfunction]
-#[pyo3(signature = (
-    raw,
-    trim_whitespace=true,
-    normalize_newlines=true,
-    min_byte_len=0,
-    max_byte_len=0,
-    truncate_over_max=false,
-    json_extract_key=None
-))]
-pub fn clean_single_record(
-    raw: &str,
-    trim_whitespace: bool,
-    normalize_newlines: bool,
-    min_byte_len: usize,
-    max_byte_len: usize,
-    truncate_over_max: bool,
-    json_extract_key: Option<String>,
-) -> Option<String> {
-    let cfg = PostProcessConfig {
-        trim_whitespace,
-        normalize_newlines,
-        min_byte_len,
-        max_byte_len,
-        truncate_over_max,
-        json_extract_key,
-    };
-    crate::postprocess::process_single_record(raw, &cfg)
-}
-
-#[pyfunction]
-#[pyo3(signature = (
-    records,
-    trim_whitespace=true,
-    normalize_newlines=true,
-    min_byte_len=0,
-    max_byte_len=0,
-    truncate_over_max=false,
-    json_extract_key=None
-))]
-#[allow(clippy::too_many_arguments)]
-pub fn clean_batch_records(
-    py: Python<'_>,
-    records: Vec<String>,
-    trim_whitespace: bool,
-    normalize_newlines: bool,
-    min_byte_len: usize,
-    max_byte_len: usize,
-    truncate_over_max: bool,
-    json_extract_key: Option<String>,
-) -> Vec<String> {
-    let cfg = PostProcessConfig {
-        trim_whitespace,
-        normalize_newlines,
-        min_byte_len,
-        max_byte_len,
-        truncate_over_max,
-        json_extract_key,
-    };
-    py.detach(|| crate::postprocess::process_batch_parallel(&records, &cfg))
-}
-
-#[pyfunction]
-#[pyo3(signature = (
-    records,
-    trim_whitespace=true,
-    normalize_newlines=true,
-    min_byte_len=1,
-    max_byte_len=0,
-    truncate_over_max=false,
-    json_extract_key=None,
-    seen_hashes=None,
-    deduplicate=true
-))]
-#[allow(clippy::too_many_arguments)]
-pub fn postprocess_and_dedup_records(
-    py: Python<'_>,
-    records: Vec<String>,
-    trim_whitespace: bool,
-    normalize_newlines: bool,
-    min_byte_len: usize,
-    max_byte_len: usize,
-    truncate_over_max: bool,
-    json_extract_key: Option<String>,
-    seen_hashes: Option<Vec<u64>>,
-    deduplicate: bool,
-) -> PyResult<(Vec<String>, Vec<u64>)> {
-    let cfg = PostProcessConfig {
-        trim_whitespace,
-        normalize_newlines,
-        min_byte_len,
-        max_byte_len,
-        truncate_over_max,
-        json_extract_key,
-    };
-    Ok(py.detach(|| {
-        let processed = crate::postprocess::process_batch_parallel_indexed(&records, &cfg);
-        if !deduplicate {
-            let cleaned_all: Vec<String> = processed.into_iter().flatten().collect();
-            return (cleaned_all, Vec::new());
-        }
-        let mut dedup = TwoStageDeduplicator::with_committed(seen_hashes.unwrap_or_default());
-        let mut cleaned_unique = Vec::with_capacity(processed.len());
-        for cleaned in processed.into_iter().flatten() {
-            if dedup.try_stage(1, cleaned.as_bytes()).is_some() {
-                cleaned_unique.push(cleaned);
-            }
-        }
-        let newly_committed = dedup.commit_batch(1);
-        (cleaned_unique, newly_committed)
-    }))
-}
-
-#[pyfunction]
-#[pyo3(signature = (
-    records,
-    trim_whitespace = true,
-    normalize_newlines = true,
-    min_byte_len = 0,
-    max_byte_len = 0,
-    truncate_over_max = false,
-    json_extract_key = None,
-    filter_quality = false,
-    min_chars = 0,
-    max_chars = 0,
-    min_words = 0,
-    min_entropy = 0.0,
-    max_repetition_ratio = 1.0,
-    max_symbol_ratio = 1.0,
-    deduplicator = None,
-    commit_dedup = false,
-))]
-#[allow(clippy::too_many_arguments)]
-pub fn fused_pipeline_process_batch(
-    py: Python<'_>,
-    records: Vec<String>,
-    trim_whitespace: bool,
-    normalize_newlines: bool,
-    min_byte_len: usize,
-    max_byte_len: usize,
-    truncate_over_max: bool,
-    json_extract_key: Option<String>,
-    filter_quality: bool,
-    min_chars: usize,
-    max_chars: usize,
-    min_words: usize,
-    min_entropy: f64,
-    max_repetition_ratio: f64,
-    max_symbol_ratio: f64,
-    deduplicator: Option<&PyStreamDeduplicator>,
-    commit_dedup: bool,
-) -> PyResult<PyStringTensor> {
-    let post_cfg = PostProcessConfig {
-        trim_whitespace,
-        normalize_newlines,
-        min_byte_len,
-        max_byte_len,
-        truncate_over_max,
-        json_extract_key,
-    };
-    let qual_cfg = crate::quality::QualityConfig {
-        min_chars,
-        max_chars,
-        min_words,
-        min_entropy,
-        max_repetition_ratio,
-        max_symbol_ratio,
-    };
-
-    let result = py.detach(|| {
-        let cleaned_candidates: Vec<String> = records
-            .par_iter()
-            .filter_map(|raw| {
-                let cleaned = crate::postprocess::process_single_record(raw, &post_cfg)?;
-                if filter_quality {
-                    let eval = crate::quality::evaluate_quality(&cleaned, &qual_cfg);
-                    if !eval.is_passed {
-                        return None;
-                    }
-                }
-                Some(cleaned)
-            })
-            .collect();
-
-        let final_strings = if let Some(dedup_wrapper) = deduplicator {
-            let mut dedup = dedup_wrapper.inner.lock();
-            let mut unique = Vec::with_capacity(cleaned_candidates.len());
-            for s in cleaned_candidates {
-                if dedup.try_stage(1, s.as_bytes()).is_some() {
-                    unique.push(s);
-                }
-            }
-            if commit_dedup {
-                dedup.commit_batch(1);
-            }
-            unique
-        } else {
-            cleaned_candidates
-        };
-
-        crate::string_tensor::StringTensor::from_strings(final_strings)
-    });
-
-    Ok(PyStringTensor { inner: result })
-}
-
-pub fn register_dataloader_module(parent_module: &Bound<'_, PyModule>) -> PyResult<()> {
-    parent_module.add_class::<PyStreamCoordinate>()?;
-    parent_module.add_class::<PyStreamDeduplicator>()?;
-    parent_module.add_class::<PyStateTracker>()?;
-    parent_module.add_class::<PyQualityFilter>()?;
-    parent_module.add_class::<PyMinHashDeduplicator>()?;
-    parent_module.add_class::<PyStringTensor>()?;
-    parent_module.add_function(wrap_pyfunction!(dataloader_cpu_isa, parent_module)?)?;
-    parent_module.add_function(wrap_pyfunction!(suggest_closest_matches, parent_module)?)?;
-    parent_module.add_function(wrap_pyfunction!(serialize_table_to_bytes, parent_module)?)?;
-    parent_module.add_function(wrap_pyfunction!(clean_single_record, parent_module)?)?;
-    parent_module.add_function(wrap_pyfunction!(clean_batch_records, parent_module)?)?;
-    parent_module.add_function(wrap_pyfunction!(
-        postprocess_and_dedup_records,
-        parent_module
-    )?)?;
-    parent_module.add_function(wrap_pyfunction!(
-        fused_pipeline_process_batch,
-        parent_module
-    )?)?;
-    Ok(())
-}
+use std::fs::File;
+use std::io::{Read, Write};
+
+use parking_lot::Mutex;
+use pyo3::exceptions::{PyRuntimeError, PyValueError};
+use pyo3::prelude::*;
+use pyo3::types::{PyBool, PyByteArray, PyBytes, PyDict, PyFloat, PyInt, PySequence, PyString};
+use rayon::prelude::*;
+
+use crate::coord::StreamCoordinate;
+use crate::dedup::TwoStageDeduplicator;
+use crate::error::{DataLoaderError, find_closest_matches};
+use crate::metadata::DualMetadataManager;
+use crate::postprocess::PostProcessConfig;
+use crate::quality::QualityConfig;
+use crate::serialize::{DataValue, classify_string_or_json, serialize_columns_by_format};
+
+fn to_py_err(err: DataLoaderError) -> PyErr {
+    match err {
+        DataLoaderError::InvalidConfig(msg) | DataLoaderError::ColumnNotFound(msg) => {
+            PyValueError::new_err(msg)
+        }
+        DataLoaderError::InvalidCoordinate => PyValueError::new_err(err.to_string()),
+        other => PyRuntimeError::new_err(other.to_string()),
+    }
+}
+
+#[pyclass(frozen, name = "StreamCoordinate")]
+#[derive(Clone, Copy)]
+pub struct PyStreamCoordinate {
+    #[pyo3(get)]
+    pub shard_id: u32,
+    #[pyo3(get)]
+    pub byte_offset: u64,
+    #[pyo3(get)]
+    pub row_group_idx: u32,
+    #[pyo3(get)]
+    pub page_idx: u32,
+    #[pyo3(get)]
+    pub row_offset_in_page: u32,
+    #[pyo3(get)]
+    pub global_row_idx: u64,
+}
+
+#[pymethods]
+impl PyStreamCoordinate {
+    #[new]
+    #[pyo3(signature = (
+        shard_id = 0,
+        byte_offset = 0,
+        row_group_idx = 0,
+        page_idx = 0,
+        row_offset_in_page = 0,
+        global_row_idx = 0,
+    ))]
+    pub fn new(
+        shard_id: u32,
+        byte_offset: u64,
+        row_group_idx: u32,
+        page_idx: u32,
+        row_offset_in_page: u32,
+        global_row_idx: u64,
+    ) -> Self {
+        Self {
+            shard_id,
+            byte_offset,
+            row_group_idx,
+            page_idx,
+            row_offset_in_page,
+            global_row_idx,
+        }
+    }
+
+    pub fn to_bytes<'py>(&self, py: Python<'py>) -> Bound<'py, PyBytes> {
+        let coord = StreamCoordinate {
+            shard_id: self.shard_id,
+            byte_offset: self.byte_offset,
+            row_group_idx: self.row_group_idx,
+            page_idx: self.page_idx,
+            row_offset_in_page: self.row_offset_in_page,
+            global_row_idx: self.global_row_idx,
+        };
+        PyBytes::new(py, &coord.to_bytes())
+    }
+
+    #[staticmethod]
+    pub fn from_bytes(raw: &[u8]) -> PyResult<Self> {
+        if raw.len() != StreamCoordinate::BYTE_LEN {
+            return Err(PyValueError::new_err(format!(
+                "Invalid coordinate length: expected {} bytes, got {}",
+                StreamCoordinate::BYTE_LEN,
+                raw.len()
+            )));
+        }
+        let arr: &[u8; StreamCoordinate::BYTE_LEN] = raw.try_into().unwrap();
+        let c = StreamCoordinate::from_bytes(arr);
+        Ok(Self {
+            shard_id: c.shard_id,
+            byte_offset: c.byte_offset,
+            row_group_idx: c.row_group_idx,
+            page_idx: c.page_idx,
+            row_offset_in_page: c.row_offset_in_page,
+            global_row_idx: c.global_row_idx,
+        })
+    }
+
+    pub fn __repr__(&self) -> String {
+        format!(
+            "StreamCoordinate(shard_id={}, byte_offset={}, row_group={}, page={}, row_offset={}, global_row={})",
+            self.shard_id,
+            self.byte_offset,
+            self.row_group_idx,
+            self.page_idx,
+            self.row_offset_in_page,
+            self.global_row_idx
+        )
+    }
+}
+
+#[pyclass(name = "StreamDeduplicator")]
+pub struct PyStreamDeduplicator {
+    inner: Mutex<TwoStageDeduplicator>,
+}
+
+#[pymethods]
+impl PyStreamDeduplicator {
+    #[new]
+    #[pyo3(signature = (initial_hashes = None))]
+    pub fn new(initial_hashes: Option<Vec<u64>>) -> Self {
+        let dedup = match initial_hashes {
+            Some(hashes) => TwoStageDeduplicator::with_committed(hashes),
+            None => TwoStageDeduplicator::new(),
+        };
+        Self {
+            inner: Mutex::new(dedup),
+        }
+    }
+
+    #[staticmethod]
+    pub fn compute_hash(payload: &[u8]) -> u64 {
+        TwoStageDeduplicator::compute_hash(payload)
+    }
+
+    pub fn is_seen(&self, payload: &[u8]) -> bool {
+        self.inner.lock().is_seen(payload)
+    }
+
+    pub fn try_add(&self, payload: &[u8]) -> bool {
+        let mut dedup = self.inner.lock();
+        dedup.try_stage(1, payload).is_some()
+    }
+
+    pub fn filter_batch(&self, records: Vec<String>) -> Vec<String> {
+        let mut dedup = self.inner.lock();
+        let mut out = Vec::with_capacity(records.len());
+        for rec in records {
+            if dedup.try_stage(1, rec.as_bytes()).is_some() {
+                out.push(rec);
+            }
+        }
+        out
+    }
+
+    pub fn commit(&self) -> usize {
+        let mut dedup = self.inner.lock();
+        dedup.commit_batch(1).len()
+    }
+
+    pub fn rollback(&self) -> usize {
+        let mut dedup = self.inner.lock();
+        dedup.rollback_batch(1)
+    }
+
+    pub fn clear(&self) {
+        let mut dedup = self.inner.lock();
+        dedup.clear_staged();
+    }
+
+    pub fn count(&self) -> usize {
+        self.inner.lock().committed_len()
+    }
+
+    pub fn save_wal(&self, path: &str) -> PyResult<usize> {
+        let dedup = self.inner.lock();
+        let hashes = dedup.export_committed_hashes();
+        let mut buf = Vec::with_capacity(hashes.len() * 8);
+        for h in &hashes {
+            buf.extend_from_slice(&h.to_le_bytes());
+        }
+        let mut file = File::create(path).map_err(|e| PyRuntimeError::new_err(e.to_string()))?;
+        file.write_all(&buf)
+            .map_err(|e| PyRuntimeError::new_err(e.to_string()))?;
+        file.sync_data()
+            .map_err(|e| PyRuntimeError::new_err(e.to_string()))?;
+        Ok(hashes.len())
+    }
+
+    pub fn load_wal(&self, path: &str) -> PyResult<usize> {
+        let mut file = match File::open(path) {
+            Ok(f) => f,
+            Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(0),
+            Err(e) => return Err(PyRuntimeError::new_err(e.to_string())),
+        };
+        let mut buf = Vec::new();
+        file.read_to_end(&mut buf)
+            .map_err(|e| PyRuntimeError::new_err(e.to_string()))?;
+        if buf.len() % 8 != 0 {
+            return Err(PyValueError::new_err(format!(
+                "Corrupted WAL file '{}': byte length {} is not a multiple of 8",
+                path,
+                buf.len()
+            )));
+        }
+        let count = buf.len() / 8;
+        let mut dedup = self.inner.lock();
+        let (chunks, _) = buf.as_chunks::<8>();
+        for chunk in chunks {
+            let h = u64::from_le_bytes(*chunk);
+            dedup.import_committed_hashes(std::iter::once(h));
+        }
+        Ok(count)
+    }
+
+    pub fn export_hashes(&self) -> Vec<u64> {
+        self.inner.lock().export_committed_hashes()
+    }
+}
+
+#[pyclass(name = "StateTracker")]
+pub struct PyStateTracker {
+    inner: Mutex<DualMetadataManager>,
+}
+
+#[pymethods]
+impl PyStateTracker {
+    #[new]
+    pub fn new(meta_path: &str) -> PyResult<Self> {
+        let mgr = DualMetadataManager::open(meta_path).map_err(to_py_err)?;
+        Ok(Self {
+            inner: Mutex::new(mgr),
+        })
+    }
+
+    pub fn read(&self) -> PyStreamCoordinate {
+        let mgr = self.inner.lock();
+        let c = mgr.committed_coordinate();
+        PyStreamCoordinate {
+            shard_id: c.shard_id,
+            byte_offset: c.byte_offset,
+            row_group_idx: c.row_group_idx,
+            page_idx: c.page_idx,
+            row_offset_in_page: c.row_offset_in_page,
+            global_row_idx: c.global_row_idx,
+        }
+    }
+
+    #[pyo3(signature = (
+        shard_id = 0,
+        byte_offset = 0,
+        row_group_idx = 0,
+        page_idx = 0,
+        row_offset_in_page = 0,
+        global_row_idx = 0,
+        batch_id = 1
+    ))]
+    #[allow(clippy::too_many_arguments)]
+    pub fn update(
+        &self,
+        shard_id: u32,
+        byte_offset: u64,
+        row_group_idx: u32,
+        page_idx: u32,
+        row_offset_in_page: u32,
+        global_row_idx: u64,
+        batch_id: u64,
+    ) {
+        let coord = StreamCoordinate {
+            shard_id,
+            byte_offset,
+            row_group_idx,
+            page_idx,
+            row_offset_in_page,
+            global_row_idx,
+        };
+        let mut mgr = self.inner.lock();
+        mgr.stage_batch_in_ram(batch_id, coord);
+    }
+
+    #[pyo3(signature = (batch_id = 1))]
+    pub fn commit(&self, batch_id: u64) -> PyResult<PyStreamCoordinate> {
+        let mut mgr = self.inner.lock();
+        let c = mgr.commit_gpu_batch(batch_id).map_err(to_py_err)?;
+        Ok(PyStreamCoordinate {
+            shard_id: c.shard_id,
+            byte_offset: c.byte_offset,
+            row_group_idx: c.row_group_idx,
+            page_idx: c.page_idx,
+            row_offset_in_page: c.row_offset_in_page,
+            global_row_idx: c.global_row_idx,
+        })
+    }
+
+    pub fn committed_batch_id(&self) -> u64 {
+        self.inner.lock().committed_batch_id()
+    }
+}
+
+#[pyclass(name = "QualityFilter")]
+pub struct PyQualityFilter {
+    cfg: QualityConfig,
+}
+
+#[pymethods]
+impl PyQualityFilter {
+    #[new]
+    #[pyo3(signature = (
+        min_chars = 0,
+        max_chars = 0,
+        min_words = 0,
+        min_entropy = 0.0,
+        max_repetition_ratio = 1.0,
+        max_symbol_ratio = 1.0
+    ))]
+    pub fn new(
+        min_chars: usize,
+        max_chars: usize,
+        min_words: usize,
+        min_entropy: f64,
+        max_repetition_ratio: f64,
+        max_symbol_ratio: f64,
+    ) -> Self {
+        Self {
+            cfg: QualityConfig {
+                min_chars,
+                max_chars,
+                min_words,
+                min_entropy,
+                max_repetition_ratio,
+                max_symbol_ratio,
+            },
+        }
+    }
+
+    pub fn is_high_quality(&self, text: &str) -> bool {
+        crate::quality::evaluate_quality(text, &self.cfg).is_passed
+    }
+
+    pub fn metrics(&self, text: &str) -> (usize, usize, f64, f64, f64, bool) {
+        let m = crate::quality::evaluate_quality(text, &self.cfg);
+        (
+            m.char_count,
+            m.word_count,
+            m.entropy,
+            m.repetition_ratio,
+            m.symbol_ratio,
+            m.is_passed,
+        )
+    }
+
+    pub fn filter_flags(&self, py: Python<'_>, texts: Vec<String>) -> Vec<bool> {
+        let cfg = self.cfg.clone();
+        py.detach(|| crate::quality::filter_quality_batch_parallel(&texts, &cfg))
+    }
+
+    pub fn filter_batch(&self, py: Python<'_>, texts: Vec<String>) -> Vec<String> {
+        let cfg = self.cfg.clone();
+        py.detach(|| {
+            texts
+                .into_iter()
+                .filter(|t| crate::quality::evaluate_quality(t, &cfg).is_passed)
+                .collect()
+        })
+    }
+}
+
+#[pyclass(name = "MinHashDeduplicator")]
+pub struct PyMinHashDeduplicator {
+    inner: Mutex<crate::minhash::MinHashDeduplicator>,
+}
+
+#[pymethods]
+impl PyMinHashDeduplicator {
+    #[new]
+    #[pyo3(signature = (threshold = 0.85, k_shingle = 5, num_perm = 64))]
+    pub fn new(threshold: f64, k_shingle: usize, num_perm: usize) -> Self {
+        Self {
+            inner: Mutex::new(crate::minhash::MinHashDeduplicator::new(
+                threshold, k_shingle, num_perm,
+            )),
+        }
+    }
+
+    pub fn is_unique(&self, text: &str) -> bool {
+        self.inner.lock().is_unique(text)
+    }
+
+    pub fn similarity(&self, text_a: &str, text_b: &str) -> f64 {
+        let dedup = self.inner.lock();
+        let sig_a = dedup.compute_signature(text_a);
+        let sig_b = dedup.compute_signature(text_b);
+        sig_a.jaccard_similarity(&sig_b)
+    }
+
+    pub fn count(&self) -> usize {
+        self.inner.lock().count()
+    }
+
+    pub fn clear(&self) {
+        self.inner.lock().clear();
+    }
+}
+
+#[pyclass(name = "StringTensor")]
+#[derive(Clone)]
+pub struct PyStringTensor {
+    inner: crate::string_tensor::StringTensor,
+}
+
+#[pymethods]
+impl PyStringTensor {
+    #[staticmethod]
+    pub fn from_list(strings: Vec<String>) -> Self {
+        Self {
+            inner: crate::string_tensor::StringTensor::from_strings(strings),
+        }
+    }
+
+    #[staticmethod]
+    pub fn from_gpu_buffers(offsets: Vec<u32>, payload: &[u8]) -> Self {
+        Self {
+            inner: crate::string_tensor::StringTensor::from_gpu_buffers(&offsets, payload),
+        }
+    }
+
+    pub fn to_gpu_buffers<'py>(
+        &self,
+        py: Python<'py>,
+    ) -> (Bound<'py, PyBytes>, Bound<'py, PyBytes>) {
+        let (offsets, payload) = self.inner.to_gpu_buffers();
+        let mut offset_bytes = Vec::with_capacity(offsets.len() * 4);
+        for &off in &offsets {
+            offset_bytes.extend_from_slice(&off.to_le_bytes());
+        }
+        (PyBytes::new(py, &offset_bytes), PyBytes::new(py, &payload))
+    }
+
+    pub fn to_offsets(&self) -> Vec<u32> {
+        self.inner.to_gpu_buffers().0
+    }
+
+    pub fn to_payload<'py>(&self, py: Python<'py>) -> Bound<'py, PyBytes> {
+        let (_, payload) = self.inner.to_gpu_buffers();
+        PyBytes::new(py, &payload)
+    }
+
+    pub fn __len__(&self) -> usize {
+        self.inner.len()
+    }
+
+    pub fn __getitem__(&self, index: usize) -> PyResult<String> {
+        self.inner
+            .get(index)
+            .map(|s| s.to_owned())
+            .ok_or_else(|| PyValueError::new_err("StringTensor index out of range"))
+    }
+
+    pub fn get(&self, index: usize) -> Option<String> {
+        self.inner.get(index).map(|s| s.to_owned())
+    }
+
+    pub fn to_list(&self) -> Vec<String> {
+        self.inner.to_vec()
+    }
+
+    pub fn lengths(&self) -> Vec<u32> {
+        self.inner.lengths()
+    }
+
+    pub fn is_inlined_mask(&self) -> Vec<bool> {
+        self.inner.is_inlined_mask()
+    }
+
+    pub fn total_bytes(&self) -> usize {
+        self.inner.views.iter().map(|v| v.len as usize).sum()
+    }
+
+    pub fn filter_by_length(&self, min_len: u32, max_len: u32) -> Self {
+        Self {
+            inner: self.inner.filter_by_length(min_len, max_len),
+        }
+    }
+
+    pub fn filter_by_prefix(&self, prefix: &str) -> Self {
+        Self {
+            inner: self.inner.filter_by_prefix(prefix),
+        }
+    }
+
+    pub fn hash_all(&self, py: Python<'_>) -> Vec<u64> {
+        py.detach(|| self.inner.hash_all_parallel())
+    }
+
+    pub fn __repr__(&self) -> String {
+        format!(
+            "StringTensor(count={}, inlined={})",
+            self.inner.len(),
+            self.inner.views.iter().filter(|v| v.is_inlined()).count()
+        )
+    }
+}
+
+pub fn extract_py_columns_table(
+    py: Python<'_>,
+    columns: &Bound<'_, PyAny>,
+) -> PyResult<Vec<(String, Vec<DataValue>)>> {
+    let mut rust_cols = Vec::new();
+
+    if let Ok(dict) = columns.cast::<PyDict>() {
+        for (k, v) in dict.iter() {
+            let col_name = k.extract::<String>()?;
+            let vals = extract_py_seq_values(py, &v)?;
+            rust_cols.push((col_name, vals));
+        }
+    } else if let Ok(seq) = columns.cast::<PySequence>() {
+        let len = seq.len()?;
+        for i in 0..len {
+            let item = seq.get_item(i)?;
+            if let Ok(pair) = item.cast::<PySequence>()
+                && pair.len()? == 2
+            {
+                let col_name = pair.get_item(0)?.extract::<String>()?;
+                let col_vals_seq = pair.get_item(1)?;
+                let vals = extract_py_seq_values(py, &col_vals_seq)?;
+                rust_cols.push((col_name, vals));
+                continue;
+            }
+            return Err(PyValueError::new_err(
+                "columns sequence must contain (name, values) pairs or be a dict",
+            ));
+        }
+    } else {
+        return Err(PyValueError::new_err(
+            "columns must be a dict {name: [values]} or sequence of (name, [values]) pairs",
+        ));
+    }
+
+    Ok(rust_cols)
+}
+
+fn extract_py_seq_values(_py: Python<'_>, seq_obj: &Bound<'_, PyAny>) -> PyResult<Vec<DataValue>> {
+    let seq = seq_obj
+        .cast::<PySequence>()
+        .map_err(|_| PyValueError::new_err("column values must be a sequence (list/tuple)"))?;
+    let len = seq.len()?;
+    let mut out = Vec::with_capacity(len);
+
+    for i in 0..len {
+        let item = seq.get_item(i)?;
+        if item.is_none() {
+            out.push(DataValue::Null);
+        } else if let Ok(b) = item.cast::<PyBool>() {
+            out.push(DataValue::Bool(b.is_true()));
+        } else if let Ok(i) = item.cast::<PyInt>() {
+            out.push(DataValue::Int(i.extract::<i64>()?));
+        } else if let Ok(f) = item.cast::<PyFloat>() {
+            out.push(DataValue::Float(f.extract::<f64>()?));
+        } else if let Ok(s) = item.cast::<PyString>() {
+            let str_val = s.to_string_lossy();
+            out.push(classify_string_or_json(&str_val));
+        } else if let Ok(bytes) = item.cast::<PyBytes>() {
+            out.push(DataValue::Bytes(bytes.as_bytes().to_vec()));
+        } else if let Ok(bytes) = item.cast::<PyByteArray>() {
+            out.push(DataValue::Bytes(bytes.to_vec()));
+        } else {
+            let py_str = item.str()?;
+            let str_repr = py_str.to_string_lossy();
+            out.push(classify_string_or_json(&str_repr));
+        }
+    }
+
+    Ok(out)
+}
+
+#[pyfunction]
+pub fn dataloader_cpu_isa() -> &'static str {
+    crate::simd_cpu::detect_cpu_isa().as_str()
+}
+
+#[pyfunction]
+#[pyo3(signature = (target, candidates, max_suggestions=3))]
+pub fn suggest_closest_matches(
+    target: &str,
+    candidates: Vec<String>,
+    max_suggestions: usize,
+) -> Vec<String> {
+    find_closest_matches(target, &candidates, max_suggestions)
+}
+
+#[pyfunction]
+#[pyo3(signature = (columns, format="parquet", hf_image_struct=true))]
+pub fn serialize_table_to_bytes(
+    py: Python<'_>,
+    columns: &Bound<'_, PyAny>,
+    format: &str,
+    hf_image_struct: bool,
+) -> PyResult<Vec<u8>> {
+    let rust_cols = extract_py_columns_table(py, columns)?;
+    py.detach(|| serialize_columns_by_format(&rust_cols, format, hf_image_struct))
+        .map_err(to_py_err)
+}
+
+#[pyfunction]
+#[pyo3(signature = (
+    raw,
+    trim_whitespace=true,
+    normalize_newlines=true,
+    min_byte_len=0,
+    max_byte_len=0,
+    truncate_over_max=false,
+    json_extract_key=None
+))]
+pub fn clean_single_record(
+    raw: &str,
+    trim_whitespace: bool,
+    normalize_newlines: bool,
+    min_byte_len: usize,
+    max_byte_len: usize,
+    truncate_over_max: bool,
+    json_extract_key: Option<String>,
+) -> Option<String> {
+    let cfg = PostProcessConfig {
+        trim_whitespace,
+        normalize_newlines,
+        min_byte_len,
+        max_byte_len,
+        truncate_over_max,
+        json_extract_key,
+    };
+    crate::postprocess::process_single_record(raw, &cfg)
+}
+
+#[pyfunction]
+#[pyo3(signature = (
+    records,
+    trim_whitespace=true,
+    normalize_newlines=true,
+    min_byte_len=0,
+    max_byte_len=0,
+    truncate_over_max=false,
+    json_extract_key=None
+))]
+#[allow(clippy::too_many_arguments)]
+pub fn clean_batch_records(
+    py: Python<'_>,
+    records: Vec<String>,
+    trim_whitespace: bool,
+    normalize_newlines: bool,
+    min_byte_len: usize,
+    max_byte_len: usize,
+    truncate_over_max: bool,
+    json_extract_key: Option<String>,
+) -> Vec<String> {
+    let cfg = PostProcessConfig {
+        trim_whitespace,
+        normalize_newlines,
+        min_byte_len,
+        max_byte_len,
+        truncate_over_max,
+        json_extract_key,
+    };
+    py.detach(|| crate::postprocess::process_batch_parallel(&records, &cfg))
+}
+
+#[pyfunction]
+#[pyo3(signature = (
+    records,
+    trim_whitespace=true,
+    normalize_newlines=true,
+    min_byte_len=1,
+    max_byte_len=0,
+    truncate_over_max=false,
+    json_extract_key=None,
+    seen_hashes=None,
+    deduplicate=true
+))]
+#[allow(clippy::too_many_arguments)]
+pub fn postprocess_and_dedup_records(
+    py: Python<'_>,
+    records: Vec<String>,
+    trim_whitespace: bool,
+    normalize_newlines: bool,
+    min_byte_len: usize,
+    max_byte_len: usize,
+    truncate_over_max: bool,
+    json_extract_key: Option<String>,
+    seen_hashes: Option<Vec<u64>>,
+    deduplicate: bool,
+) -> PyResult<(Vec<String>, Vec<u64>)> {
+    let cfg = PostProcessConfig {
+        trim_whitespace,
+        normalize_newlines,
+        min_byte_len,
+        max_byte_len,
+        truncate_over_max,
+        json_extract_key,
+    };
+    Ok(py.detach(|| {
+        let processed = crate::postprocess::process_batch_parallel_indexed(&records, &cfg);
+        if !deduplicate {
+            let cleaned_all: Vec<String> = processed.into_iter().flatten().collect();
+            return (cleaned_all, Vec::new());
+        }
+        let mut dedup = TwoStageDeduplicator::with_committed(seen_hashes.unwrap_or_default());
+        let mut cleaned_unique = Vec::with_capacity(processed.len());
+        for cleaned in processed.into_iter().flatten() {
+            if dedup.try_stage(1, cleaned.as_bytes()).is_some() {
+                cleaned_unique.push(cleaned);
+            }
+        }
+        let newly_committed = dedup.commit_batch(1);
+        (cleaned_unique, newly_committed)
+    }))
+}
+
+#[pyfunction]
+#[pyo3(signature = (
+    records,
+    trim_whitespace = true,
+    normalize_newlines = true,
+    min_byte_len = 0,
+    max_byte_len = 0,
+    truncate_over_max = false,
+    json_extract_key = None,
+    filter_quality = false,
+    min_chars = 0,
+    max_chars = 0,
+    min_words = 0,
+    min_entropy = 0.0,
+    max_repetition_ratio = 1.0,
+    max_symbol_ratio = 1.0,
+    deduplicator = None,
+    commit_dedup = false,
+))]
+#[allow(clippy::too_many_arguments)]
+pub fn fused_pipeline_process_batch(
+    py: Python<'_>,
+    records: Vec<String>,
+    trim_whitespace: bool,
+    normalize_newlines: bool,
+    min_byte_len: usize,
+    max_byte_len: usize,
+    truncate_over_max: bool,
+    json_extract_key: Option<String>,
+    filter_quality: bool,
+    min_chars: usize,
+    max_chars: usize,
+    min_words: usize,
+    min_entropy: f64,
+    max_repetition_ratio: f64,
+    max_symbol_ratio: f64,
+    deduplicator: Option<&PyStreamDeduplicator>,
+    commit_dedup: bool,
+) -> PyResult<PyStringTensor> {
+    let post_cfg = PostProcessConfig {
+        trim_whitespace,
+        normalize_newlines,
+        min_byte_len,
+        max_byte_len,
+        truncate_over_max,
+        json_extract_key,
+    };
+    let qual_cfg = crate::quality::QualityConfig {
+        min_chars,
+        max_chars,
+        min_words,
+        min_entropy,
+        max_repetition_ratio,
+        max_symbol_ratio,
+    };
+
+    let result = py.detach(|| {
+        let cleaned_candidates: Vec<String> = records
+            .par_iter()
+            .filter_map(|raw| {
+                let cleaned = crate::postprocess::process_single_record(raw, &post_cfg)?;
+                if filter_quality {
+                    let eval = crate::quality::evaluate_quality(&cleaned, &qual_cfg);
+                    if !eval.is_passed {
+                        return None;
+                    }
+                }
+                Some(cleaned)
+            })
+            .collect();
+
+        let final_strings = if let Some(dedup_wrapper) = deduplicator {
+            let mut dedup = dedup_wrapper.inner.lock();
+            let mut unique = Vec::with_capacity(cleaned_candidates.len());
+            for s in cleaned_candidates {
+                if dedup.try_stage(1, s.as_bytes()).is_some() {
+                    unique.push(s);
+                }
+            }
+            if commit_dedup {
+                dedup.commit_batch(1);
+            }
+            unique
+        } else {
+            cleaned_candidates
+        };
+
+        crate::string_tensor::StringTensor::from_strings(final_strings)
+    });
+
+    Ok(PyStringTensor { inner: result })
+}
+
+pub fn register_dataloader_module(parent_module: &Bound<'_, PyModule>) -> PyResult<()> {
+    parent_module.add_class::<PyStreamCoordinate>()?;
+    parent_module.add_class::<PyStreamDeduplicator>()?;
+    parent_module.add_class::<PyStateTracker>()?;
+    parent_module.add_class::<PyQualityFilter>()?;
+    parent_module.add_class::<PyMinHashDeduplicator>()?;
+    parent_module.add_class::<PyStringTensor>()?;
+    parent_module.add_function(wrap_pyfunction!(dataloader_cpu_isa, parent_module)?)?;
+    parent_module.add_function(wrap_pyfunction!(suggest_closest_matches, parent_module)?)?;
+    parent_module.add_function(wrap_pyfunction!(serialize_table_to_bytes, parent_module)?)?;
+    parent_module.add_function(wrap_pyfunction!(clean_single_record, parent_module)?)?;
+    parent_module.add_function(wrap_pyfunction!(clean_batch_records, parent_module)?)?;
+    parent_module.add_function(wrap_pyfunction!(
+        postprocess_and_dedup_records,
+        parent_module
+    )?)?;
+    parent_module.add_function(wrap_pyfunction!(
+        fused_pipeline_process_batch,
+        parent_module
+    )?)?;
+    Ok(())
+}

data/src/quality.rs

@@ -1,169 +1,169 @@
-use hashbrown::{HashMap, HashSet};
-use rayon::prelude::*;
-
-#[derive(Debug, Clone)]
-pub struct QualityConfig {
-    pub min_chars: usize,
-    pub max_chars: usize,
-    pub min_words: usize,
-    pub min_entropy: f64,
-    pub max_repetition_ratio: f64,
-    pub max_symbol_ratio: f64,
-}
-
-impl Default for QualityConfig {
-    fn default() -> Self {
-        Self {
-            min_chars: 0,
-            max_chars: 0,
-            min_words: 0,
-            min_entropy: 0.0,
-            max_repetition_ratio: 1.0,
-            max_symbol_ratio: 1.0,
-        }
-    }
-}
-
-#[derive(Debug, Clone, Copy, PartialEq)]
-pub struct QualityMetrics {
-    pub char_count: usize,
-    pub word_count: usize,
-    pub entropy: f64,
-    pub repetition_ratio: f64,
-    pub symbol_ratio: f64,
-    pub is_passed: bool,
-}
-
-pub fn calculate_shannon_entropy(text: &str) -> f64 {
-    if text.is_empty() {
-        return 0.0;
-    }
-
-    // Zero-allocation stack array fast-path for pure ASCII text
-    if text.is_ascii() {
-        let mut freq = [0u32; 256];
-        for &b in text.as_bytes() {
-            freq[b as usize] += 1;
-        }
-
-        let n = text.len() as f64;
-        let mut entropy = 0.0;
-        for &count in &freq {
-            if count > 0 {
-                let p = (count as f64) / n;
-                entropy -= p * p.log2();
-            }
-        }
-        return entropy;
-    }
-
-    let char_count = text.chars().count();
-    let n = char_count as f64;
-    let mut freq = HashMap::new();
-    for c in text.chars() {
-        *freq.entry(c).or_insert(0usize) += 1;
-    }
-    let mut entropy = 0.0;
-    for &count in freq.values() {
-        let p = (count as f64) / n;
-        entropy -= p * p.log2();
-    }
-    entropy
-}
-
-pub fn calculate_repetition_ratio(words: &[&str]) -> f64 {
-    if words.len() < 4 {
-        return 0.0;
-    }
-
-    let total_trigrams = words.len() - 2;
-    let mut unique_trigrams = HashSet::with_capacity(total_trigrams);
-
-    for w in words.windows(3) {
-        unique_trigrams.insert((w[0], w[1], w[2]));
-    }
-
-    1.0 - ((unique_trigrams.len() as f64) / (total_trigrams as f64))
-}
-
-pub fn calculate_symbol_ratio(text: &str) -> (usize, f64) {
-    let mut symbols = 0usize;
-    let mut visible = 0usize;
-
-    for c in text.chars() {
-        if !c.is_whitespace() {
-            visible += 1;
-            if !c.is_alphanumeric() {
-                symbols += 1;
-            }
-        }
-    }
-
-    if visible == 0 {
-        return (0, 0.0);
-    }
-
-    (visible, (symbols as f64) / (visible as f64))
-}
-
-pub fn evaluate_quality(text: &str, cfg: &QualityConfig) -> QualityMetrics {
-    let char_count = text.chars().count();
-    if cfg.min_chars > 0 && char_count < cfg.min_chars {
-        return QualityMetrics {
-            char_count,
-            word_count: 0,
-            entropy: 0.0,
-            repetition_ratio: 0.0,
-            symbol_ratio: 0.0,
-            is_passed: false,
-        };
-    }
-
-    if cfg.max_chars > 0 && char_count > cfg.max_chars {
-        return QualityMetrics {
-            char_count,
-            word_count: 0,
-            entropy: 0.0,
-            repetition_ratio: 0.0,
-            symbol_ratio: 0.0,
-            is_passed: false,
-        };
-    }
-
-    let words: Vec<&str> = text.split_whitespace().collect();
-    let word_count = words.len();
-    if cfg.min_words > 0 && word_count < cfg.min_words {
-        return QualityMetrics {
-            char_count,
-            word_count,
-            entropy: 0.0,
-            repetition_ratio: 0.0,
-            symbol_ratio: 0.0,
-            is_passed: false,
-        };
-    }
-
-    let entropy = calculate_shannon_entropy(text);
-    let repetition_ratio = calculate_repetition_ratio(&words);
-    let (_, symbol_ratio) = calculate_symbol_ratio(text);
-
-    let is_passed = (cfg.min_entropy <= 0.0 || entropy >= cfg.min_entropy)
-        && (cfg.max_repetition_ratio >= 1.0 || repetition_ratio <= cfg.max_repetition_ratio)
-        && (cfg.max_symbol_ratio >= 1.0 || symbol_ratio <= cfg.max_symbol_ratio);
-
-    QualityMetrics {
-        char_count,
-        word_count,
-        entropy,
-        repetition_ratio,
-        symbol_ratio,
-        is_passed,
-    }
-}
-
-pub fn filter_quality_batch_parallel(texts: &[String], cfg: &QualityConfig) -> Vec<bool> {
-    texts
-        .par_iter()
-        .map(|t| evaluate_quality(t, cfg).is_passed)
-        .collect()
-}
+use hashbrown::{HashMap, HashSet};
+use rayon::prelude::*;
+
+#[derive(Debug, Clone)]
+pub struct QualityConfig {
+    pub min_chars: usize,
+    pub max_chars: usize,
+    pub min_words: usize,
+    pub min_entropy: f64,
+    pub max_repetition_ratio: f64,
+    pub max_symbol_ratio: f64,
+}
+
+impl Default for QualityConfig {
+    fn default() -> Self {
+        Self {
+            min_chars: 0,
+            max_chars: 0,
+            min_words: 0,
+            min_entropy: 0.0,
+            max_repetition_ratio: 1.0,
+            max_symbol_ratio: 1.0,
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub struct QualityMetrics {
+    pub char_count: usize,
+    pub word_count: usize,
+    pub entropy: f64,
+    pub repetition_ratio: f64,
+    pub symbol_ratio: f64,
+    pub is_passed: bool,
+}
+
+pub fn calculate_shannon_entropy(text: &str) -> f64 {
+    if text.is_empty() {
+        return 0.0;
+    }
+
+    // Zero-allocation stack array fast-path for pure ASCII text
+    if text.is_ascii() {
+        let mut freq = [0u32; 256];
+        for &b in text.as_bytes() {
+            freq[b as usize] += 1;
+        }
+
+        let n = text.len() as f64;
+        let mut entropy = 0.0;
+        for &count in &freq {
+            if count > 0 {
+                let p = (count as f64) / n;
+                entropy -= p * p.log2();
+            }
+        }
+        return entropy;
+    }
+
+    let char_count = text.chars().count();
+    let n = char_count as f64;
+    let mut freq = HashMap::new();
+    for c in text.chars() {
+        *freq.entry(c).or_insert(0usize) += 1;
+    }
+    let mut entropy = 0.0;
+    for &count in freq.values() {
+        let p = (count as f64) / n;
+        entropy -= p * p.log2();
+    }
+    entropy
+}
+
+pub fn calculate_repetition_ratio(words: &[&str]) -> f64 {
+    if words.len() < 4 {
+        return 0.0;
+    }
+
+    let total_trigrams = words.len() - 2;
+    let mut unique_trigrams = HashSet::with_capacity(total_trigrams);
+
+    for w in words.windows(3) {
+        unique_trigrams.insert((w[0], w[1], w[2]));
+    }
+
+    1.0 - ((unique_trigrams.len() as f64) / (total_trigrams as f64))
+}
+
+pub fn calculate_symbol_ratio(text: &str) -> (usize, f64) {
+    let mut symbols = 0usize;
+    let mut visible = 0usize;
+
+    for c in text.chars() {
+        if !c.is_whitespace() {
+            visible += 1;
+            if !c.is_alphanumeric() {
+                symbols += 1;
+            }
+        }
+    }
+
+    if visible == 0 {
+        return (0, 0.0);
+    }
+
+    (visible, (symbols as f64) / (visible as f64))
+}
+
+pub fn evaluate_quality(text: &str, cfg: &QualityConfig) -> QualityMetrics {
+    let char_count = text.chars().count();
+    if cfg.min_chars > 0 && char_count < cfg.min_chars {
+        return QualityMetrics {
+            char_count,
+            word_count: 0,
+            entropy: 0.0,
+            repetition_ratio: 0.0,
+            symbol_ratio: 0.0,
+            is_passed: false,
+        };
+    }
+
+    if cfg.max_chars > 0 && char_count > cfg.max_chars {
+        return QualityMetrics {
+            char_count,
+            word_count: 0,
+            entropy: 0.0,
+            repetition_ratio: 0.0,
+            symbol_ratio: 0.0,
+            is_passed: false,
+        };
+    }
+
+    let words: Vec<&str> = text.split_whitespace().collect();
+    let word_count = words.len();
+    if cfg.min_words > 0 && word_count < cfg.min_words {
+        return QualityMetrics {
+            char_count,
+            word_count,
+            entropy: 0.0,
+            repetition_ratio: 0.0,
+            symbol_ratio: 0.0,
+            is_passed: false,
+        };
+    }
+
+    let entropy = calculate_shannon_entropy(text);
+    let repetition_ratio = calculate_repetition_ratio(&words);
+    let (_, symbol_ratio) = calculate_symbol_ratio(text);
+
+    let is_passed = (cfg.min_entropy <= 0.0 || entropy >= cfg.min_entropy)
+        && (cfg.max_repetition_ratio >= 1.0 || repetition_ratio <= cfg.max_repetition_ratio)
+        && (cfg.max_symbol_ratio >= 1.0 || symbol_ratio <= cfg.max_symbol_ratio);
+
+    QualityMetrics {
+        char_count,
+        word_count,
+        entropy,
+        repetition_ratio,
+        symbol_ratio,
+        is_passed,
+    }
+}
+
+pub fn filter_quality_batch_parallel(texts: &[String], cfg: &QualityConfig) -> Vec<bool> {
+    texts
+        .par_iter()
+        .map(|t| evaluate_quality(t, cfg).is_passed)
+        .collect()
+}

data/src/serialize.rs

@@ -1,268 +1,268 @@
-use std::sync::Arc;
-
-use arrow_array::builder::{
-    BinaryBuilder, BooleanBuilder, Float64Builder, Int64Builder, StringBuilder,
-};
-use arrow_array::{Array, RecordBatch, StructArray};
-use arrow_ipc::writer::FileWriter as ArrowIpcFileWriter;
-use arrow_schema::{DataType, Field, Fields, Schema};
-use parquet::arrow::ArrowWriter;
-use parquet::basic::Compression;
-use parquet::file::properties::WriterProperties;
-
-use crate::error::{DataLoaderError, Result};
-
-#[derive(Debug, Clone, PartialEq)]
-pub enum DataValue {
-    Null,
-    Text(String),
-    Bytes(Vec<u8>),
-    Json(String),
-    Int(i64),
-    Float(f64),
-    Bool(bool),
-    List(Vec<DataValue>),
-}
-
-pub fn classify_string_or_json(s: &str) -> DataValue {
-    let trimmed = s.trim();
-    if ((trimmed.starts_with('{') && trimmed.ends_with('}'))
-        || (trimmed.starts_with('[') && trimmed.ends_with(']')))
-        && serde_json::from_str::<serde_json::Value>(trimmed).is_ok()
-    {
-        return DataValue::Json(trimmed.to_owned());
-    }
-    DataValue::Text(s.to_owned())
-}
-
-pub fn build_arrow_record_batch(
-    columns: &[(String, Vec<DataValue>)],
-    hf_image_struct: bool,
-) -> Result<(Arc<Schema>, RecordBatch)> {
-    if columns.is_empty() {
-        return Err(DataLoaderError::InvalidConfig(
-            "Cannot serialize empty column list".to_owned(),
-        ));
-    }
-
-    let num_rows = columns[0].1.len();
-    let mut fields = Vec::with_capacity(columns.len());
-    let mut arrays: Vec<Arc<dyn Array>> = Vec::with_capacity(columns.len());
-
-    for (name, vals) in columns {
-        if vals.len() != num_rows {
-            return Err(DataLoaderError::InvalidConfig(format!(
-                "Column '{name}' length mismatch: expected {num_rows}, got {}",
-                vals.len()
-            )));
-        }
-
-        let has_non_null = vals.iter().any(|v| !matches!(v, DataValue::Null));
-        let is_binary = vals.iter().any(|v| matches!(v, DataValue::Bytes(_)));
-        let all_int = has_non_null
-            && vals
-                .iter()
-                .all(|v| matches!(v, DataValue::Null | DataValue::Int(_)));
-        let has_float = vals.iter().any(|v| matches!(v, DataValue::Float(_)));
-        let all_numeric = has_non_null
-            && has_float
-            && vals
-                .iter()
-                .all(|v| matches!(v, DataValue::Null | DataValue::Int(_) | DataValue::Float(_)));
-        let all_bool = has_non_null
-            && vals
-                .iter()
-                .all(|v| matches!(v, DataValue::Null | DataValue::Bool(_)));
-
-        if is_binary {
-            if hf_image_struct {
-                let mut bytes_builder = BinaryBuilder::with_capacity(num_rows, num_rows * 1024);
-                let mut path_builder = StringBuilder::with_capacity(num_rows, num_rows * 16);
-
-                for val in vals {
-                    match val {
-                        DataValue::Bytes(b) => {
-                            bytes_builder.append_value(b);
-                            path_builder.append_null();
-                        }
-                        DataValue::Text(t) => {
-                            bytes_builder.append_null();
-                            path_builder.append_value(t);
-                        }
-                        _ => {
-                            bytes_builder.append_null();
-                            path_builder.append_null();
-                        }
-                    }
-                }
-
-                let bytes_arr = Arc::new(bytes_builder.finish()) as Arc<dyn Array>;
-                let path_arr = Arc::new(path_builder.finish()) as Arc<dyn Array>;
-
-                let struct_fields = Fields::from(vec![
-                    Field::new("bytes", DataType::Binary, true),
-                    Field::new("path", DataType::Utf8, true),
-                ]);
-
-                let struct_arr =
-                    StructArray::new(struct_fields.clone(), vec![bytes_arr, path_arr], None);
-
-                fields.push(Field::new(name, DataType::Struct(struct_fields), true));
-                arrays.push(Arc::new(struct_arr));
-            } else {
-                let mut b_builder = BinaryBuilder::with_capacity(num_rows, num_rows * 1024);
-                for val in vals {
-                    match val {
-                        DataValue::Bytes(b) => b_builder.append_value(b),
-                        _ => b_builder.append_null(),
-                    }
-                }
-                fields.push(Field::new(name, DataType::Binary, true));
-                arrays.push(Arc::new(b_builder.finish()));
-            }
-        } else if all_int {
-            let mut i_builder = Int64Builder::with_capacity(num_rows);
-            for val in vals {
-                match val {
-                    DataValue::Int(i) => i_builder.append_value(*i),
-                    _ => i_builder.append_null(),
-                }
-            }
-            fields.push(Field::new(name, DataType::Int64, true));
-            arrays.push(Arc::new(i_builder.finish()));
-        } else if all_numeric {
-            let mut f_builder = Float64Builder::with_capacity(num_rows);
-            for val in vals {
-                match val {
-                    DataValue::Float(f) => f_builder.append_value(*f),
-                    DataValue::Int(i) => f_builder.append_value(*i as f64),
-                    _ => f_builder.append_null(),
-                }
-            }
-            fields.push(Field::new(name, DataType::Float64, true));
-            arrays.push(Arc::new(f_builder.finish()));
-        } else if all_bool {
-            let mut b_builder = BooleanBuilder::with_capacity(num_rows);
-            for val in vals {
-                match val {
-                    DataValue::Bool(b) => b_builder.append_value(*b),
-                    _ => b_builder.append_null(),
-                }
-            }
-            fields.push(Field::new(name, DataType::Boolean, true));
-            arrays.push(Arc::new(b_builder.finish()));
-        } else {
-            let mut s_builder = StringBuilder::with_capacity(num_rows, num_rows * 64);
-            for val in vals {
-                match val {
-                    DataValue::Null => s_builder.append_null(),
-                    DataValue::Text(s) | DataValue::Json(s) => s_builder.append_value(s),
-                    DataValue::Int(i) => s_builder.append_value(i.to_string()),
-                    DataValue::Float(f) => s_builder.append_value(f.to_string()),
-                    DataValue::Bool(b) => s_builder.append_value(b.to_string()),
-                    DataValue::Bytes(b) => s_builder.append_value(format!("<{} bytes>", b.len())),
-                    DataValue::List(l) => {
-                        let json = serde_json::to_string(l).unwrap_or_default();
-                        s_builder.append_value(json);
-                    }
-                }
-            }
-            fields.push(Field::new(name, DataType::Utf8, true));
-            arrays.push(Arc::new(s_builder.finish()));
-        }
-    }
-
-    let schema = Arc::new(Schema::new(fields));
-    let batch = RecordBatch::try_new(schema.clone(), arrays)?;
-    Ok((schema, batch))
-}
-
-impl serde::Serialize for DataValue {
-    fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
-    where
-        S: serde::Serializer,
-    {
-        match self {
-            DataValue::Null => serializer.serialize_none(),
-            DataValue::Text(s) => serializer.serialize_str(s),
-            DataValue::Bytes(b) => serializer.serialize_bytes(b),
-            DataValue::Json(j) => {
-                if let Ok(v) = serde_json::from_str::<serde_json::Value>(j) {
-                    v.serialize(serializer)
-                } else {
-                    serializer.serialize_str(j)
-                }
-            }
-            DataValue::Int(i) => serializer.serialize_i64(*i),
-            DataValue::Float(f) => serializer.serialize_f64(*f),
-            DataValue::Bool(b) => serializer.serialize_bool(*b),
-            DataValue::List(l) => l.serialize(serializer),
-        }
-    }
-}
-
-pub fn serialize_columns_to_parquet_bytes(
-    columns: &[(String, Vec<DataValue>)],
-    hf_image_struct: bool,
-) -> Result<Vec<u8>> {
-    let (schema, batch) = build_arrow_record_batch(columns, hf_image_struct)?;
-    let mut out = Vec::new();
-    let props = WriterProperties::builder()
-        .set_compression(Compression::ZSTD(Default::default()))
-        .build();
-    let mut writer = ArrowWriter::try_new(&mut out, schema, Some(props))?;
-    writer.write(&batch)?;
-    writer.close()?;
-    Ok(out)
-}
-
-pub fn serialize_columns_to_arrow_ipc_bytes(
-    columns: &[(String, Vec<DataValue>)],
-    hf_image_struct: bool,
-) -> Result<Vec<u8>> {
-    let (schema, batch) = build_arrow_record_batch(columns, hf_image_struct)?;
-    let mut out = Vec::new();
-    let mut writer = ArrowIpcFileWriter::try_new(&mut out, &schema)?;
-    writer.write(&batch)?;
-    writer.finish()?;
-    Ok(out)
-}
-
-pub fn serialize_columns_to_jsonl_bytes(columns: &[(String, Vec<DataValue>)]) -> Result<Vec<u8>> {
-    if columns.is_empty() {
-        return Ok(Vec::new());
-    }
-    let num_rows = columns[0].1.len();
-    let mut out = Vec::new();
-    for row_idx in 0..num_rows {
-        let mut map = serde_json::Map::with_capacity(columns.len());
-        for (col_name, vals) in columns {
-            let val = vals.get(row_idx).unwrap_or(&DataValue::Null);
-            let json_val = serde_json::to_value(val).unwrap_or(serde_json::Value::Null);
-            map.insert(col_name.clone(), json_val);
-        }
-        let line = serde_json::to_string(&map)
-            .map_err(|e| DataLoaderError::InvalidConfig(e.to_string()))?;
-        out.extend_from_slice(line.as_bytes());
-        out.push(b'\n');
-    }
-    Ok(out)
-}
-
-pub fn serialize_columns_by_format(
-    columns: &[(String, Vec<DataValue>)],
-    format: &str,
-    hf_image_struct: bool,
-) -> Result<Vec<u8>> {
-    let lower = format.trim().to_ascii_lowercase();
-    if lower == "parquet" || lower.ends_with(".parquet") || lower.ends_with(".pq") {
-        serialize_columns_to_parquet_bytes(columns, hf_image_struct)
-    } else if lower == "arrow" || lower == "feather" || lower == "ipc" || lower.ends_with(".arrow")
-    {
-        serialize_columns_to_arrow_ipc_bytes(columns, hf_image_struct)
-    } else if lower == "jsonl" || lower == "ndjson" || lower.ends_with(".jsonl") {
-        serialize_columns_to_jsonl_bytes(columns)
-    } else {
-        serialize_columns_to_parquet_bytes(columns, hf_image_struct)
-    }
-}
+use std::sync::Arc;
+
+use arrow_array::builder::{
+    BinaryBuilder, BooleanBuilder, Float64Builder, Int64Builder, StringBuilder,
+};
+use arrow_array::{Array, RecordBatch, StructArray};
+use arrow_ipc::writer::FileWriter as ArrowIpcFileWriter;
+use arrow_schema::{DataType, Field, Fields, Schema};
+use parquet::arrow::ArrowWriter;
+use parquet::basic::Compression;
+use parquet::file::properties::WriterProperties;
+
+use crate::error::{DataLoaderError, Result};
+
+#[derive(Debug, Clone, PartialEq)]
+pub enum DataValue {
+    Null,
+    Text(String),
+    Bytes(Vec<u8>),
+    Json(String),
+    Int(i64),
+    Float(f64),
+    Bool(bool),
+    List(Vec<DataValue>),
+}
+
+pub fn classify_string_or_json(s: &str) -> DataValue {
+    let trimmed = s.trim();
+    if ((trimmed.starts_with('{') && trimmed.ends_with('}'))
+        || (trimmed.starts_with('[') && trimmed.ends_with(']')))
+        && serde_json::from_str::<serde_json::Value>(trimmed).is_ok()
+    {
+        return DataValue::Json(trimmed.to_owned());
+    }
+    DataValue::Text(s.to_owned())
+}
+
+pub fn build_arrow_record_batch(
+    columns: &[(String, Vec<DataValue>)],
+    hf_image_struct: bool,
+) -> Result<(Arc<Schema>, RecordBatch)> {
+    if columns.is_empty() {
+        return Err(DataLoaderError::InvalidConfig(
+            "Cannot serialize empty column list".to_owned(),
+        ));
+    }
+
+    let num_rows = columns[0].1.len();
+    let mut fields = Vec::with_capacity(columns.len());
+    let mut arrays: Vec<Arc<dyn Array>> = Vec::with_capacity(columns.len());
+
+    for (name, vals) in columns {
+        if vals.len() != num_rows {
+            return Err(DataLoaderError::InvalidConfig(format!(
+                "Column '{name}' length mismatch: expected {num_rows}, got {}",
+                vals.len()
+            )));
+        }
+
+        let has_non_null = vals.iter().any(|v| !matches!(v, DataValue::Null));
+        let is_binary = vals.iter().any(|v| matches!(v, DataValue::Bytes(_)));
+        let all_int = has_non_null
+            && vals
+                .iter()
+                .all(|v| matches!(v, DataValue::Null | DataValue::Int(_)));
+        let has_float = vals.iter().any(|v| matches!(v, DataValue::Float(_)));
+        let all_numeric = has_non_null
+            && has_float
+            && vals
+                .iter()
+                .all(|v| matches!(v, DataValue::Null | DataValue::Int(_) | DataValue::Float(_)));
+        let all_bool = has_non_null
+            && vals
+                .iter()
+                .all(|v| matches!(v, DataValue::Null | DataValue::Bool(_)));
+
+        if is_binary {
+            if hf_image_struct {
+                let mut bytes_builder = BinaryBuilder::with_capacity(num_rows, num_rows * 1024);
+                let mut path_builder = StringBuilder::with_capacity(num_rows, num_rows * 16);
+
+                for val in vals {
+                    match val {
+                        DataValue::Bytes(b) => {
+                            bytes_builder.append_value(b);
+                            path_builder.append_null();
+                        }
+                        DataValue::Text(t) => {
+                            bytes_builder.append_null();
+                            path_builder.append_value(t);
+                        }
+                        _ => {
+                            bytes_builder.append_null();
+                            path_builder.append_null();
+                        }
+                    }
+                }
+
+                let bytes_arr = Arc::new(bytes_builder.finish()) as Arc<dyn Array>;
+                let path_arr = Arc::new(path_builder.finish()) as Arc<dyn Array>;
+
+                let struct_fields = Fields::from(vec![
+                    Field::new("bytes", DataType::Binary, true),
+                    Field::new("path", DataType::Utf8, true),
+                ]);
+
+                let struct_arr =
+                    StructArray::new(struct_fields.clone(), vec![bytes_arr, path_arr], None);
+
+                fields.push(Field::new(name, DataType::Struct(struct_fields), true));
+                arrays.push(Arc::new(struct_arr));
+            } else {
+                let mut b_builder = BinaryBuilder::with_capacity(num_rows, num_rows * 1024);
+                for val in vals {
+                    match val {
+                        DataValue::Bytes(b) => b_builder.append_value(b),
+                        _ => b_builder.append_null(),
+                    }
+                }
+                fields.push(Field::new(name, DataType::Binary, true));
+                arrays.push(Arc::new(b_builder.finish()));
+            }
+        } else if all_int {
+            let mut i_builder = Int64Builder::with_capacity(num_rows);
+            for val in vals {
+                match val {
+                    DataValue::Int(i) => i_builder.append_value(*i),
+                    _ => i_builder.append_null(),
+                }
+            }
+            fields.push(Field::new(name, DataType::Int64, true));
+            arrays.push(Arc::new(i_builder.finish()));
+        } else if all_numeric {
+            let mut f_builder = Float64Builder::with_capacity(num_rows);
+            for val in vals {
+                match val {
+                    DataValue::Float(f) => f_builder.append_value(*f),
+                    DataValue::Int(i) => f_builder.append_value(*i as f64),
+                    _ => f_builder.append_null(),
+                }
+            }
+            fields.push(Field::new(name, DataType::Float64, true));
+            arrays.push(Arc::new(f_builder.finish()));
+        } else if all_bool {
+            let mut b_builder = BooleanBuilder::with_capacity(num_rows);
+            for val in vals {
+                match val {
+                    DataValue::Bool(b) => b_builder.append_value(*b),
+                    _ => b_builder.append_null(),
+                }
+            }
+            fields.push(Field::new(name, DataType::Boolean, true));
+            arrays.push(Arc::new(b_builder.finish()));
+        } else {
+            let mut s_builder = StringBuilder::with_capacity(num_rows, num_rows * 64);
+            for val in vals {
+                match val {
+                    DataValue::Null => s_builder.append_null(),
+                    DataValue::Text(s) | DataValue::Json(s) => s_builder.append_value(s),
+                    DataValue::Int(i) => s_builder.append_value(i.to_string()),
+                    DataValue::Float(f) => s_builder.append_value(f.to_string()),
+                    DataValue::Bool(b) => s_builder.append_value(b.to_string()),
+                    DataValue::Bytes(b) => s_builder.append_value(format!("<{} bytes>", b.len())),
+                    DataValue::List(l) => {
+                        let json = serde_json::to_string(l).unwrap_or_default();
+                        s_builder.append_value(json);
+                    }
+                }
+            }
+            fields.push(Field::new(name, DataType::Utf8, true));
+            arrays.push(Arc::new(s_builder.finish()));
+        }
+    }
+
+    let schema = Arc::new(Schema::new(fields));
+    let batch = RecordBatch::try_new(schema.clone(), arrays)?;
+    Ok((schema, batch))
+}
+
+impl serde::Serialize for DataValue {
+    fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        match self {
+            DataValue::Null => serializer.serialize_none(),
+            DataValue::Text(s) => serializer.serialize_str(s),
+            DataValue::Bytes(b) => serializer.serialize_bytes(b),
+            DataValue::Json(j) => {
+                if let Ok(v) = serde_json::from_str::<serde_json::Value>(j) {
+                    v.serialize(serializer)
+                } else {
+                    serializer.serialize_str(j)
+                }
+            }
+            DataValue::Int(i) => serializer.serialize_i64(*i),
+            DataValue::Float(f) => serializer.serialize_f64(*f),
+            DataValue::Bool(b) => serializer.serialize_bool(*b),
+            DataValue::List(l) => l.serialize(serializer),
+        }
+    }
+}
+
+pub fn serialize_columns_to_parquet_bytes(
+    columns: &[(String, Vec<DataValue>)],
+    hf_image_struct: bool,
+) -> Result<Vec<u8>> {
+    let (schema, batch) = build_arrow_record_batch(columns, hf_image_struct)?;
+    let mut out = Vec::new();
+    let props = WriterProperties::builder()
+        .set_compression(Compression::ZSTD(Default::default()))
+        .build();
+    let mut writer = ArrowWriter::try_new(&mut out, schema, Some(props))?;
+    writer.write(&batch)?;
+    writer.close()?;
+    Ok(out)
+}
+
+pub fn serialize_columns_to_arrow_ipc_bytes(
+    columns: &[(String, Vec<DataValue>)],
+    hf_image_struct: bool,
+) -> Result<Vec<u8>> {
+    let (schema, batch) = build_arrow_record_batch(columns, hf_image_struct)?;
+    let mut out = Vec::new();
+    let mut writer = ArrowIpcFileWriter::try_new(&mut out, &schema)?;
+    writer.write(&batch)?;
+    writer.finish()?;
+    Ok(out)
+}
+
+pub fn serialize_columns_to_jsonl_bytes(columns: &[(String, Vec<DataValue>)]) -> Result<Vec<u8>> {
+    if columns.is_empty() {
+        return Ok(Vec::new());
+    }
+    let num_rows = columns[0].1.len();
+    let mut out = Vec::new();
+    for row_idx in 0..num_rows {
+        let mut map = serde_json::Map::with_capacity(columns.len());
+        for (col_name, vals) in columns {
+            let val = vals.get(row_idx).unwrap_or(&DataValue::Null);
+            let json_val = serde_json::to_value(val).unwrap_or(serde_json::Value::Null);
+            map.insert(col_name.clone(), json_val);
+        }
+        let line = serde_json::to_string(&map)
+            .map_err(|e| DataLoaderError::InvalidConfig(e.to_string()))?;
+        out.extend_from_slice(line.as_bytes());
+        out.push(b'\n');
+    }
+    Ok(out)
+}
+
+pub fn serialize_columns_by_format(
+    columns: &[(String, Vec<DataValue>)],
+    format: &str,
+    hf_image_struct: bool,
+) -> Result<Vec<u8>> {
+    let lower = format.trim().to_ascii_lowercase();
+    if lower == "parquet" || lower.ends_with(".parquet") || lower.ends_with(".pq") {
+        serialize_columns_to_parquet_bytes(columns, hf_image_struct)
+    } else if lower == "arrow" || lower == "feather" || lower == "ipc" || lower.ends_with(".arrow")
+    {
+        serialize_columns_to_arrow_ipc_bytes(columns, hf_image_struct)
+    } else if lower == "jsonl" || lower == "ndjson" || lower.ends_with(".jsonl") {
+        serialize_columns_to_jsonl_bytes(columns)
+    } else {
+        serialize_columns_to_parquet_bytes(columns, hf_image_struct)
+    }
+}

data/src/simd_cpu.rs

@@ -1,313 +1,313 @@
-use std::sync::OnceLock;
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum CpuIsa {
-    Avx512,
-    Avx2,
-    Neon,
-    Scalar,
-}
-
-impl CpuIsa {
-    pub fn as_str(self) -> &'static str {
-        match self {
-            Self::Avx512 => "AVX-512",
-            Self::Avx2 => "AVX2",
-            Self::Neon => "NEON",
-            Self::Scalar => "SCALAR",
-        }
-    }
-}
-
-#[repr(align(64))]
-#[derive(Debug, Clone)]
-pub struct CacheAlignedBuffer {
-    pub data: Vec<u8>,
-}
-
-impl CacheAlignedBuffer {
-    pub fn with_capacity(cap: usize) -> Self {
-        let aligned_cap = (cap + 63) & !63;
-        Self {
-            data: Vec::with_capacity(aligned_cap),
-        }
-    }
-}
-
-#[inline]
-pub fn detect_cpu_isa() -> CpuIsa {
-    static CACHED_ISA: OnceLock<CpuIsa> = OnceLock::new();
-    *CACHED_ISA.get_or_init(|| {
-        std::cfg_select! {
-            any(target_arch = "x86", target_arch = "x86_64") => {
-                if std::arch::is_x86_feature_detected!("avx512f")
-                    && std::arch::is_x86_feature_detected!("avx512bw")
-                {
-                    CpuIsa::Avx512
-                } else if std::arch::is_x86_feature_detected!("avx2") {
-                    CpuIsa::Avx2
-                } else {
-                    CpuIsa::Scalar
-                }
-            }
-            target_arch = "aarch64" => {
-                if std::arch::is_aarch64_feature_detected!("neon") {
-                    CpuIsa::Neon
-                } else {
-                    CpuIsa::Scalar
-                }
-            }
-            _ => {
-                CpuIsa::Scalar
-            }
-        }
-    })
-}
-
-#[inline]
-pub fn find_byte_fast(haystack: &[u8], needle: u8) -> Option<usize> {
-    #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
-    {
-        match detect_cpu_isa() {
-            CpuIsa::Avx512 => return unsafe { find_byte_avx512(haystack, needle) },
-            CpuIsa::Avx2 => return unsafe { find_byte_avx2(haystack, needle) },
-            _ => {}
-        }
-    }
-    #[cfg(target_arch = "aarch64")]
-    {
-        if detect_cpu_isa() == CpuIsa::Neon {
-            return unsafe { find_byte_neon(haystack, needle) };
-        }
-    }
-    haystack.iter().position(|&b| b == needle)
-}
-
-#[inline]
-pub fn rfind_byte_fast(haystack: &[u8], needle: u8) -> Option<usize> {
-    #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
-    {
-        match detect_cpu_isa() {
-            CpuIsa::Avx512 => return unsafe { rfind_byte_avx512(haystack, needle) },
-            CpuIsa::Avx2 => return unsafe { rfind_byte_avx2(haystack, needle) },
-            _ => {}
-        }
-    }
-    #[cfg(target_arch = "aarch64")]
-    {
-        if detect_cpu_isa() == CpuIsa::Neon {
-            return unsafe { rfind_byte_neon(haystack, needle) };
-        }
-    }
-    haystack.iter().rposition(|&b| b == needle)
-}
-
-#[inline]
-pub fn contains_byte_fast(haystack: &[u8], needle: u8) -> bool {
-    find_byte_fast(haystack, needle).is_some()
-}
-
-#[inline]
-pub fn find_newline_fast(haystack: &[u8]) -> Option<usize> {
-    find_byte_fast(haystack, b'\n')
-}
-
-#[inline]
-pub fn rfind_newline_fast(haystack: &[u8]) -> Option<usize> {
-    rfind_byte_fast(haystack, b'\n')
-}
-
-#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
-#[target_feature(enable = "avx512f,avx512bw")]
-unsafe fn find_byte_avx512(haystack: &[u8], needle: u8) -> Option<usize> {
-    #[cfg(target_arch = "x86")]
-    use std::arch::x86::*;
-    #[cfg(target_arch = "x86_64")]
-    use std::arch::x86_64::*;
-
-    let len = haystack.len();
-    let ptr = haystack.as_ptr();
-    let v_needle = _mm512_set1_epi8(needle as i8);
-    let mut offset = 0usize;
-
-    while offset + 64 <= len {
-        let chunk = unsafe { _mm512_loadu_si512(ptr.add(offset).cast()) };
-        let mask = _mm512_cmpeq_epi8_mask(chunk, v_needle);
-        if mask != 0 {
-            return Some(offset + mask.trailing_zeros() as usize);
-        }
-        offset += 64;
-    }
-
-    while offset < len {
-        if unsafe { *ptr.add(offset) } == needle {
-            return Some(offset);
-        }
-        offset += 1;
-    }
-    None
-}
-
-#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
-#[target_feature(enable = "avx512f,avx512bw")]
-unsafe fn rfind_byte_avx512(haystack: &[u8], needle: u8) -> Option<usize> {
-    #[cfg(target_arch = "x86")]
-    use std::arch::x86::*;
-    #[cfg(target_arch = "x86_64")]
-    use std::arch::x86_64::*;
-
-    let len = haystack.len();
-    let ptr = haystack.as_ptr();
-    let v_needle = _mm512_set1_epi8(needle as i8);
-    let mut end = len;
-
-    while end >= 64 {
-        let start = end - 64;
-        let chunk = unsafe { _mm512_loadu_si512(ptr.add(start).cast()) };
-        let mask = _mm512_cmpeq_epi8_mask(chunk, v_needle);
-        if mask != 0 {
-            let highest_bit = 63 - mask.leading_zeros() as usize;
-            return Some(start + highest_bit);
-        }
-        end = start;
-    }
-
-    while end > 0 {
-        end -= 1;
-        if unsafe { *ptr.add(end) } == needle {
-            return Some(end);
-        }
-    }
-    None
-}
-
-#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
-#[target_feature(enable = "avx2")]
-unsafe fn find_byte_avx2(haystack: &[u8], needle: u8) -> Option<usize> {
-    #[cfg(target_arch = "x86")]
-    use std::arch::x86::*;
-    #[cfg(target_arch = "x86_64")]
-    use std::arch::x86_64::*;
-
-    let len = haystack.len();
-    let ptr = haystack.as_ptr();
-    let v_needle = _mm256_set1_epi8(needle as i8);
-    let mut offset = 0usize;
-
-    while offset + 32 <= len {
-        let chunk = unsafe { _mm256_loadu_si256(ptr.add(offset).cast()) };
-        let cmp = _mm256_cmpeq_epi8(chunk, v_needle);
-        let mask = _mm256_movemask_epi8(cmp) as u32;
-        if mask != 0 {
-            return Some(offset + mask.trailing_zeros() as usize);
-        }
-        offset += 32;
-    }
-
-    while offset < len {
-        if unsafe { *ptr.add(offset) } == needle {
-            return Some(offset);
-        }
-        offset += 1;
-    }
-    None
-}
-
-#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
-#[target_feature(enable = "avx2")]
-unsafe fn rfind_byte_avx2(haystack: &[u8], needle: u8) -> Option<usize> {
-    #[cfg(target_arch = "x86")]
-    use std::arch::x86::*;
-    #[cfg(target_arch = "x86_64")]
-    use std::arch::x86_64::*;
-
-    let len = haystack.len();
-    let ptr = haystack.as_ptr();
-    let v_needle = _mm256_set1_epi8(needle as i8);
-    let mut end = len;
-
-    while end >= 32 {
-        let start = end - 32;
-        let chunk = unsafe { _mm256_loadu_si256(ptr.add(start).cast()) };
-        let cmp = _mm256_cmpeq_epi8(chunk, v_needle);
-        let mask = _mm256_movemask_epi8(cmp) as u32;
-        if mask != 0 {
-            let highest_bit = 31 - mask.leading_zeros() as usize;
-            return Some(start + highest_bit);
-        }
-        end = start;
-    }
-
-    while end > 0 {
-        end -= 1;
-        if unsafe { *ptr.add(end) } == needle {
-            return Some(end);
-        }
-    }
-    None
-}
-
-#[cfg(target_arch = "aarch64")]
-#[target_feature(enable = "neon")]
-unsafe fn find_byte_neon(haystack: &[u8], needle: u8) -> Option<usize> {
-    use std::arch::aarch64::*;
-
-    let len = haystack.len();
-    let ptr = haystack.as_ptr();
-    let v_needle = vdupq_n_u8(needle);
-    let mut offset = 0usize;
-
-    while offset + 16 <= len {
-        let chunk = unsafe { vld1q_u8(ptr.add(offset)) };
-        let cmp = vceqq_u8(chunk, v_needle);
-        if vmaxvq_u8(cmp) != 0 {
-            for lane in 0..16 {
-                if unsafe { *ptr.add(offset + lane) } == needle {
-                    return Some(offset + lane);
-                }
-            }
-        }
-        offset += 16;
-    }
-
-    while offset < len {
-        if unsafe { *ptr.add(offset) } == needle {
-            return Some(offset);
-        }
-        offset += 1;
-    }
-    None
-}
-
-#[cfg(target_arch = "aarch64")]
-#[target_feature(enable = "neon")]
-unsafe fn rfind_byte_neon(haystack: &[u8], needle: u8) -> Option<usize> {
-    use std::arch::aarch64::*;
-
-    let len = haystack.len();
-    let ptr = haystack.as_ptr();
-    let v_needle = vdupq_n_u8(needle);
-    let mut end = len;
-
-    while end >= 16 {
-        let start = end - 16;
-        let chunk = unsafe { vld1q_u8(ptr.add(start)) };
-        let cmp = vceqq_u8(chunk, v_needle);
-        if vmaxvq_u8(cmp) != 0 {
-            for lane in (0..16).rev() {
-                if unsafe { *ptr.add(start + lane) } == needle {
-                    return Some(start + lane);
-                }
-            }
-        }
-        end = start;
-    }
-
-    while end > 0 {
-        end -= 1;
-        if unsafe { *ptr.add(end) } == needle {
-            return Some(end);
-        }
-    }
-    None
-}
+use std::sync::OnceLock;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum CpuIsa {
+    Avx512,
+    Avx2,
+    Neon,
+    Scalar,
+}
+
+impl CpuIsa {
+    pub fn as_str(self) -> &'static str {
+        match self {
+            Self::Avx512 => "AVX-512",
+            Self::Avx2 => "AVX2",
+            Self::Neon => "NEON",
+            Self::Scalar => "SCALAR",
+        }
+    }
+}
+
+#[repr(align(64))]
+#[derive(Debug, Clone)]
+pub struct CacheAlignedBuffer {
+    pub data: Vec<u8>,
+}
+
+impl CacheAlignedBuffer {
+    pub fn with_capacity(cap: usize) -> Self {
+        let aligned_cap = (cap + 63) & !63;
+        Self {
+            data: Vec::with_capacity(aligned_cap),
+        }
+    }
+}
+
+#[inline]
+pub fn detect_cpu_isa() -> CpuIsa {
+    static CACHED_ISA: OnceLock<CpuIsa> = OnceLock::new();
+    *CACHED_ISA.get_or_init(|| {
+        std::cfg_select! {
+            any(target_arch = "x86", target_arch = "x86_64") => {
+                if std::arch::is_x86_feature_detected!("avx512f")
+                    && std::arch::is_x86_feature_detected!("avx512bw")
+                {
+                    CpuIsa::Avx512
+                } else if std::arch::is_x86_feature_detected!("avx2") {
+                    CpuIsa::Avx2
+                } else {
+                    CpuIsa::Scalar
+                }
+            }
+            target_arch = "aarch64" => {
+                if std::arch::is_aarch64_feature_detected!("neon") {
+                    CpuIsa::Neon
+                } else {
+                    CpuIsa::Scalar
+                }
+            }
+            _ => {
+                CpuIsa::Scalar
+            }
+        }
+    })
+}
+
+#[inline]
+pub fn find_byte_fast(haystack: &[u8], needle: u8) -> Option<usize> {
+    #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+    {
+        match detect_cpu_isa() {
+            CpuIsa::Avx512 => return unsafe { find_byte_avx512(haystack, needle) },
+            CpuIsa::Avx2 => return unsafe { find_byte_avx2(haystack, needle) },
+            _ => {}
+        }
+    }
+    #[cfg(target_arch = "aarch64")]
+    {
+        if detect_cpu_isa() == CpuIsa::Neon {
+            return unsafe { find_byte_neon(haystack, needle) };
+        }
+    }
+    haystack.iter().position(|&b| b == needle)
+}
+
+#[inline]
+pub fn rfind_byte_fast(haystack: &[u8], needle: u8) -> Option<usize> {
+    #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+    {
+        match detect_cpu_isa() {
+            CpuIsa::Avx512 => return unsafe { rfind_byte_avx512(haystack, needle) },
+            CpuIsa::Avx2 => return unsafe { rfind_byte_avx2(haystack, needle) },
+            _ => {}
+        }
+    }
+    #[cfg(target_arch = "aarch64")]
+    {
+        if detect_cpu_isa() == CpuIsa::Neon {
+            return unsafe { rfind_byte_neon(haystack, needle) };
+        }
+    }
+    haystack.iter().rposition(|&b| b == needle)
+}
+
+#[inline]
+pub fn contains_byte_fast(haystack: &[u8], needle: u8) -> bool {
+    find_byte_fast(haystack, needle).is_some()
+}
+
+#[inline]
+pub fn find_newline_fast(haystack: &[u8]) -> Option<usize> {
+    find_byte_fast(haystack, b'\n')
+}
+
+#[inline]
+pub fn rfind_newline_fast(haystack: &[u8]) -> Option<usize> {
+    rfind_byte_fast(haystack, b'\n')
+}
+
+#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+#[target_feature(enable = "avx512f,avx512bw")]
+unsafe fn find_byte_avx512(haystack: &[u8], needle: u8) -> Option<usize> {
+    #[cfg(target_arch = "x86")]
+    use std::arch::x86::*;
+    #[cfg(target_arch = "x86_64")]
+    use std::arch::x86_64::*;
+
+    let len = haystack.len();
+    let ptr = haystack.as_ptr();
+    let v_needle = _mm512_set1_epi8(needle as i8);
+    let mut offset = 0usize;
+
+    while offset + 64 <= len {
+        let chunk = unsafe { _mm512_loadu_si512(ptr.add(offset).cast()) };
+        let mask = _mm512_cmpeq_epi8_mask(chunk, v_needle);
+        if mask != 0 {
+            return Some(offset + mask.trailing_zeros() as usize);
+        }
+        offset += 64;
+    }
+
+    while offset < len {
+        if unsafe { *ptr.add(offset) } == needle {
+            return Some(offset);
+        }
+        offset += 1;
+    }
+    None
+}
+
+#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+#[target_feature(enable = "avx512f,avx512bw")]
+unsafe fn rfind_byte_avx512(haystack: &[u8], needle: u8) -> Option<usize> {
+    #[cfg(target_arch = "x86")]
+    use std::arch::x86::*;
+    #[cfg(target_arch = "x86_64")]
+    use std::arch::x86_64::*;
+
+    let len = haystack.len();
+    let ptr = haystack.as_ptr();
+    let v_needle = _mm512_set1_epi8(needle as i8);
+    let mut end = len;
+
+    while end >= 64 {
+        let start = end - 64;
+        let chunk = unsafe { _mm512_loadu_si512(ptr.add(start).cast()) };
+        let mask = _mm512_cmpeq_epi8_mask(chunk, v_needle);
+        if mask != 0 {
+            let highest_bit = 63 - mask.leading_zeros() as usize;
+            return Some(start + highest_bit);
+        }
+        end = start;
+    }
+
+    while end > 0 {
+        end -= 1;
+        if unsafe { *ptr.add(end) } == needle {
+            return Some(end);
+        }
+    }
+    None
+}
+
+#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+#[target_feature(enable = "avx2")]
+unsafe fn find_byte_avx2(haystack: &[u8], needle: u8) -> Option<usize> {
+    #[cfg(target_arch = "x86")]
+    use std::arch::x86::*;
+    #[cfg(target_arch = "x86_64")]
+    use std::arch::x86_64::*;
+
+    let len = haystack.len();
+    let ptr = haystack.as_ptr();
+    let v_needle = _mm256_set1_epi8(needle as i8);
+    let mut offset = 0usize;
+
+    while offset + 32 <= len {
+        let chunk = unsafe { _mm256_loadu_si256(ptr.add(offset).cast()) };
+        let cmp = _mm256_cmpeq_epi8(chunk, v_needle);
+        let mask = _mm256_movemask_epi8(cmp) as u32;
+        if mask != 0 {
+            return Some(offset + mask.trailing_zeros() as usize);
+        }
+        offset += 32;
+    }
+
+    while offset < len {
+        if unsafe { *ptr.add(offset) } == needle {
+            return Some(offset);
+        }
+        offset += 1;
+    }
+    None
+}
+
+#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+#[target_feature(enable = "avx2")]
+unsafe fn rfind_byte_avx2(haystack: &[u8], needle: u8) -> Option<usize> {
+    #[cfg(target_arch = "x86")]
+    use std::arch::x86::*;
+    #[cfg(target_arch = "x86_64")]
+    use std::arch::x86_64::*;
+
+    let len = haystack.len();
+    let ptr = haystack.as_ptr();
+    let v_needle = _mm256_set1_epi8(needle as i8);
+    let mut end = len;
+
+    while end >= 32 {
+        let start = end - 32;
+        let chunk = unsafe { _mm256_loadu_si256(ptr.add(start).cast()) };
+        let cmp = _mm256_cmpeq_epi8(chunk, v_needle);
+        let mask = _mm256_movemask_epi8(cmp) as u32;
+        if mask != 0 {
+            let highest_bit = 31 - mask.leading_zeros() as usize;
+            return Some(start + highest_bit);
+        }
+        end = start;
+    }
+
+    while end > 0 {
+        end -= 1;
+        if unsafe { *ptr.add(end) } == needle {
+            return Some(end);
+        }
+    }
+    None
+}
+
+#[cfg(target_arch = "aarch64")]
+#[target_feature(enable = "neon")]
+unsafe fn find_byte_neon(haystack: &[u8], needle: u8) -> Option<usize> {
+    use std::arch::aarch64::*;
+
+    let len = haystack.len();
+    let ptr = haystack.as_ptr();
+    let v_needle = vdupq_n_u8(needle);
+    let mut offset = 0usize;
+
+    while offset + 16 <= len {
+        let chunk = unsafe { vld1q_u8(ptr.add(offset)) };
+        let cmp = vceqq_u8(chunk, v_needle);
+        if vmaxvq_u8(cmp) != 0 {
+            for lane in 0..16 {
+                if unsafe { *ptr.add(offset + lane) } == needle {
+                    return Some(offset + lane);
+                }
+            }
+        }
+        offset += 16;
+    }
+
+    while offset < len {
+        if unsafe { *ptr.add(offset) } == needle {
+            return Some(offset);
+        }
+        offset += 1;
+    }
+    None
+}
+
+#[cfg(target_arch = "aarch64")]
+#[target_feature(enable = "neon")]
+unsafe fn rfind_byte_neon(haystack: &[u8], needle: u8) -> Option<usize> {
+    use std::arch::aarch64::*;
+
+    let len = haystack.len();
+    let ptr = haystack.as_ptr();
+    let v_needle = vdupq_n_u8(needle);
+    let mut end = len;
+
+    while end >= 16 {
+        let start = end - 16;
+        let chunk = unsafe { vld1q_u8(ptr.add(start)) };
+        let cmp = vceqq_u8(chunk, v_needle);
+        if vmaxvq_u8(cmp) != 0 {
+            for lane in (0..16).rev() {
+                if unsafe { *ptr.add(start + lane) } == needle {
+                    return Some(start + lane);
+                }
+            }
+        }
+        end = start;
+    }
+
+    while end > 0 {
+        end -= 1;
+        if unsafe { *ptr.add(end) } == needle {
+            return Some(end);
+        }
+    }
+    None
+}

data/src/string_tensor.rs

@@ -1,269 +1,269 @@
-use std::sync::Arc;
-
-use rayon::prelude::*;
-use xxhash_rust::xxh3::xxh3_64;
-
-#[repr(C)]
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
-pub struct StringView16 {
-    pub(crate) len: u32,
-    pub(crate) prefix_or_inline: [u8; 4],
-    pub(crate) suffix_or_offset: [u8; 8],
-}
-
-impl StringView16 {
-    pub const INLINE_CAPACITY: usize = 12;
-
-    #[inline]
-    pub fn new(s: &str, buffer_id: u32, offset: u32) -> Self {
-        let len = s.len() as u32;
-        let bytes = s.as_bytes();
-        let mut prefix_or_inline = [0u8; 4];
-        let mut suffix_or_offset = [0u8; 8];
-
-        if (len as usize) <= Self::INLINE_CAPACITY {
-            let n = bytes.len();
-            let first_part = n.min(4);
-            prefix_or_inline[..first_part].copy_from_slice(&bytes[..first_part]);
-            if n > 4 {
-                suffix_or_offset[..n - 4].copy_from_slice(&bytes[4..n]);
-            }
-        } else {
-            prefix_or_inline.copy_from_slice(&bytes[..4]);
-            suffix_or_offset[0..4].copy_from_slice(&offset.to_le_bytes());
-            suffix_or_offset[4..8].copy_from_slice(&buffer_id.to_le_bytes());
-        }
-
-        Self {
-            len,
-            prefix_or_inline,
-            suffix_or_offset,
-        }
-    }
-
-    #[inline]
-    pub fn is_inlined(&self) -> bool {
-        (self.len as usize) <= Self::INLINE_CAPACITY
-    }
-
-    #[inline]
-    pub fn as_str<'a>(&'a self, pool: &'a [u8]) -> &'a str {
-        if self.is_inlined() {
-            let len = (self.len as usize).min(Self::INLINE_CAPACITY);
-            // The 12 inline bytes immediately follow the 4-byte len in memory
-            unsafe {
-                let ptr = (self as *const Self as *const u8).add(4);
-                let slice = std::slice::from_raw_parts(ptr, len);
-                std::str::from_utf8_unchecked(slice)
-            }
-        } else {
-            let offset =
-                u32::from_le_bytes(self.suffix_or_offset[0..4].try_into().unwrap()) as usize;
-            let len = self.len as usize;
-            if let Some(slice) = pool.get(offset..offset + len) {
-                unsafe { std::str::from_utf8_unchecked(slice) }
-            } else {
-                ""
-            }
-        }
-    }
-
-    #[inline]
-    pub fn fast_eq<'a>(&'a self, other: &'a Self, pool: &'a [u8]) -> bool {
-        if self.len != other.len {
-            return false;
-        }
-        if self.prefix_or_inline != other.prefix_or_inline {
-            return false;
-        }
-        if self.is_inlined() {
-            return self.suffix_or_offset == other.suffix_or_offset;
-        }
-        self.as_str(pool) == other.as_str(pool)
-    }
-
-    #[inline]
-    pub fn compute_hash<'a>(&'a self, pool: &'a [u8]) -> u64 {
-        let s = self.as_str(pool);
-        xxh3_64(s.as_bytes())
-    }
-}
-
-#[derive(Clone, Debug, Default)]
-pub struct StringTensor {
-    pub views: Vec<StringView16>,
-    pub pool: Arc<Vec<u8>>,
-}
-
-impl StringTensor {
-    pub fn new() -> Self {
-        Self {
-            views: Vec::new(),
-            pool: Arc::new(Vec::new()),
-        }
-    }
-
-    pub fn from_strings<I, S>(strings: I) -> Self
-    where
-        I: IntoIterator<Item = S>,
-        S: AsRef<str>,
-    {
-        let items: Vec<S> = strings.into_iter().collect();
-        let mut total_ref_bytes = 0usize;
-        for s in &items {
-            let str_ref = s.as_ref();
-            if str_ref.len() > StringView16::INLINE_CAPACITY {
-                total_ref_bytes += str_ref.len();
-            }
-        }
-
-        let mut pool = Vec::with_capacity(total_ref_bytes);
-        let mut views = Vec::with_capacity(items.len());
-
-        for s in items {
-            let str_ref = s.as_ref();
-            if str_ref.len() <= StringView16::INLINE_CAPACITY {
-                views.push(StringView16::new(str_ref, 0, 0));
-            } else {
-                let offset = u32::try_from(pool.len()).unwrap_or(u32::MAX);
-                pool.extend_from_slice(str_ref.as_bytes());
-                views.push(StringView16::new(str_ref, 0, offset));
-            }
-        }
-
-        Self {
-            views,
-            pool: Arc::new(pool),
-        }
-    }
-
-    pub fn from_gpu_buffers(offsets: &[u32], payload: &[u8]) -> Self {
-        if offsets.is_empty() {
-            return Self::new();
-        }
-
-        let num_strings = offsets.len().saturating_sub(1);
-        let mut views = Vec::with_capacity(num_strings);
-        let pool = Arc::new(payload.to_vec());
-
-        for i in 0..num_strings {
-            let start = offsets[i] as usize;
-            let end = offsets[i + 1] as usize;
-            if start <= end && end <= payload.len() {
-                if let Ok(s) = std::str::from_utf8(&payload[start..end]) {
-                    views.push(StringView16::new(s, 0, start as u32));
-                } else {
-                    views.push(StringView16::new("", 0, 0));
-                }
-            } else {
-                views.push(StringView16::new("", 0, 0));
-            }
-        }
-
-        Self { views, pool }
-    }
-
-    pub fn to_gpu_buffers(&self) -> (Vec<u32>, Vec<u8>) {
-        let num_strings = self.views.len();
-        let mut offsets = Vec::with_capacity(num_strings + 1);
-        let mut total_chars = 0usize;
-
-        for v in &self.views {
-            total_chars += v.len as usize;
-        }
-
-        let mut payload = Vec::with_capacity(total_chars);
-        let mut current_offset = 0u32;
-        offsets.push(current_offset);
-
-        for v in &self.views {
-            let s = v.as_str(&self.pool);
-            payload.extend_from_slice(s.as_bytes());
-            current_offset = current_offset.saturating_add(v.len);
-            offsets.push(current_offset);
-        }
-
-        (offsets, payload)
-    }
-
-    #[inline]
-    pub fn len(&self) -> usize {
-        self.views.len()
-    }
-
-    #[inline]
-    pub fn is_empty(&self) -> bool {
-        self.views.is_empty()
-    }
-
-    #[inline]
-    pub fn get(&self, index: usize) -> Option<&str> {
-        self.views.get(index).map(|v| v.as_str(&self.pool))
-    }
-
-    pub fn to_vec(&self) -> Vec<String> {
-        self.views
-            .iter()
-            .map(|v| v.as_str(&self.pool).to_owned())
-            .collect()
-    }
-
-    pub fn lengths(&self) -> Vec<u32> {
-        self.views.iter().map(|v| v.len).collect()
-    }
-
-    pub fn is_inlined_mask(&self) -> Vec<bool> {
-        self.views.iter().map(|v| v.is_inlined()).collect()
-    }
-
-    pub fn hash_all_parallel(&self) -> Vec<u64> {
-        let pool = &self.pool;
-        self.views
-            .par_iter()
-            .map(|v| v.compute_hash(pool))
-            .collect()
-    }
-
-    pub fn filter_by_length(&self, min_len: u32, max_len: u32) -> Self {
-        let mut new_views = Vec::new();
-        for v in &self.views {
-            let l = v.len;
-            if l >= min_len && (max_len == 0 || l <= max_len) {
-                new_views.push(*v);
-            }
-        }
-        Self {
-            views: new_views,
-            pool: self.pool.clone(),
-        }
-    }
-
-    pub fn filter_by_prefix(&self, prefix: &str) -> Self {
-        let prefix_bytes = prefix.as_bytes();
-        let pool = &self.pool;
-        let mut new_views = Vec::new();
-
-        for v in &self.views {
-            let l = v.len as usize;
-            if l < prefix_bytes.len() {
-                continue;
-            }
-            // Fast prefix check
-            if prefix_bytes.len() <= 4 {
-                if v.prefix_or_inline[..prefix_bytes.len()] == prefix_bytes[..] {
-                    new_views.push(*v);
-                }
-            } else if v.prefix_or_inline[..4] == prefix_bytes[..4] {
-                let s = v.as_str(pool);
-                if s.as_bytes().starts_with(prefix_bytes) {
-                    new_views.push(*v);
-                }
-            }
-        }
-
-        Self {
-            views: new_views,
-            pool: self.pool.clone(),
-        }
-    }
-}
+use std::sync::Arc;
+
+use rayon::prelude::*;
+use xxhash_rust::xxh3::xxh3_64;
+
+#[repr(C)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub struct StringView16 {
+    pub(crate) len: u32,
+    pub(crate) prefix_or_inline: [u8; 4],
+    pub(crate) suffix_or_offset: [u8; 8],
+}
+
+impl StringView16 {
+    pub const INLINE_CAPACITY: usize = 12;
+
+    #[inline]
+    pub fn new(s: &str, buffer_id: u32, offset: u32) -> Self {
+        let len = s.len() as u32;
+        let bytes = s.as_bytes();
+        let mut prefix_or_inline = [0u8; 4];
+        let mut suffix_or_offset = [0u8; 8];
+
+        if (len as usize) <= Self::INLINE_CAPACITY {
+            let n = bytes.len();
+            let first_part = n.min(4);
+            prefix_or_inline[..first_part].copy_from_slice(&bytes[..first_part]);
+            if n > 4 {
+                suffix_or_offset[..n - 4].copy_from_slice(&bytes[4..n]);
+            }
+        } else {
+            prefix_or_inline.copy_from_slice(&bytes[..4]);
+            suffix_or_offset[0..4].copy_from_slice(&offset.to_le_bytes());
+            suffix_or_offset[4..8].copy_from_slice(&buffer_id.to_le_bytes());
+        }
+
+        Self {
+            len,
+            prefix_or_inline,
+            suffix_or_offset,
+        }
+    }
+
+    #[inline]
+    pub fn is_inlined(&self) -> bool {
+        (self.len as usize) <= Self::INLINE_CAPACITY
+    }
+
+    #[inline]
+    pub fn as_str<'a>(&'a self, pool: &'a [u8]) -> &'a str {
+        if self.is_inlined() {
+            let len = (self.len as usize).min(Self::INLINE_CAPACITY);
+            // The 12 inline bytes immediately follow the 4-byte len in memory
+            unsafe {
+                let ptr = (self as *const Self as *const u8).add(4);
+                let slice = std::slice::from_raw_parts(ptr, len);
+                std::str::from_utf8_unchecked(slice)
+            }
+        } else {
+            let offset =
+                u32::from_le_bytes(self.suffix_or_offset[0..4].try_into().unwrap()) as usize;
+            let len = self.len as usize;
+            if let Some(slice) = pool.get(offset..offset + len) {
+                unsafe { std::str::from_utf8_unchecked(slice) }
+            } else {
+                ""
+            }
+        }
+    }
+
+    #[inline]
+    pub fn fast_eq<'a>(&'a self, other: &'a Self, pool: &'a [u8]) -> bool {
+        if self.len != other.len {
+            return false;
+        }
+        if self.prefix_or_inline != other.prefix_or_inline {
+            return false;
+        }
+        if self.is_inlined() {
+            return self.suffix_or_offset == other.suffix_or_offset;
+        }
+        self.as_str(pool) == other.as_str(pool)
+    }
+
+    #[inline]
+    pub fn compute_hash<'a>(&'a self, pool: &'a [u8]) -> u64 {
+        let s = self.as_str(pool);
+        xxh3_64(s.as_bytes())
+    }
+}
+
+#[derive(Clone, Debug, Default)]
+pub struct StringTensor {
+    pub views: Vec<StringView16>,
+    pub pool: Arc<Vec<u8>>,
+}
+
+impl StringTensor {
+    pub fn new() -> Self {
+        Self {
+            views: Vec::new(),
+            pool: Arc::new(Vec::new()),
+        }
+    }
+
+    pub fn from_strings<I, S>(strings: I) -> Self
+    where
+        I: IntoIterator<Item = S>,
+        S: AsRef<str>,
+    {
+        let items: Vec<S> = strings.into_iter().collect();
+        let mut total_ref_bytes = 0usize;
+        for s in &items {
+            let str_ref = s.as_ref();
+            if str_ref.len() > StringView16::INLINE_CAPACITY {
+                total_ref_bytes += str_ref.len();
+            }
+        }
+
+        let mut pool = Vec::with_capacity(total_ref_bytes);
+        let mut views = Vec::with_capacity(items.len());
+
+        for s in items {
+            let str_ref = s.as_ref();
+            if str_ref.len() <= StringView16::INLINE_CAPACITY {
+                views.push(StringView16::new(str_ref, 0, 0));
+            } else {
+                let offset = u32::try_from(pool.len()).unwrap_or(u32::MAX);
+                pool.extend_from_slice(str_ref.as_bytes());
+                views.push(StringView16::new(str_ref, 0, offset));
+            }
+        }
+
+        Self {
+            views,
+            pool: Arc::new(pool),
+        }
+    }
+
+    pub fn from_gpu_buffers(offsets: &[u32], payload: &[u8]) -> Self {
+        if offsets.is_empty() {
+            return Self::new();
+        }
+
+        let num_strings = offsets.len().saturating_sub(1);
+        let mut views = Vec::with_capacity(num_strings);
+        let pool = Arc::new(payload.to_vec());
+
+        for i in 0..num_strings {
+            let start = offsets[i] as usize;
+            let end = offsets[i + 1] as usize;
+            if start <= end && end <= payload.len() {
+                if let Ok(s) = std::str::from_utf8(&payload[start..end]) {
+                    views.push(StringView16::new(s, 0, start as u32));
+                } else {
+                    views.push(StringView16::new("", 0, 0));
+                }
+            } else {
+                views.push(StringView16::new("", 0, 0));
+            }
+        }
+
+        Self { views, pool }
+    }
+
+    pub fn to_gpu_buffers(&self) -> (Vec<u32>, Vec<u8>) {
+        let num_strings = self.views.len();
+        let mut offsets = Vec::with_capacity(num_strings + 1);
+        let mut total_chars = 0usize;
+
+        for v in &self.views {
+            total_chars += v.len as usize;
+        }
+
+        let mut payload = Vec::with_capacity(total_chars);
+        let mut current_offset = 0u32;
+        offsets.push(current_offset);
+
+        for v in &self.views {
+            let s = v.as_str(&self.pool);
+            payload.extend_from_slice(s.as_bytes());
+            current_offset = current_offset.saturating_add(v.len);
+            offsets.push(current_offset);
+        }
+
+        (offsets, payload)
+    }
+
+    #[inline]
+    pub fn len(&self) -> usize {
+        self.views.len()
+    }
+
+    #[inline]
+    pub fn is_empty(&self) -> bool {
+        self.views.is_empty()
+    }
+
+    #[inline]
+    pub fn get(&self, index: usize) -> Option<&str> {
+        self.views.get(index).map(|v| v.as_str(&self.pool))
+    }
+
+    pub fn to_vec(&self) -> Vec<String> {
+        self.views
+            .iter()
+            .map(|v| v.as_str(&self.pool).to_owned())
+            .collect()
+    }
+
+    pub fn lengths(&self) -> Vec<u32> {
+        self.views.iter().map(|v| v.len).collect()
+    }
+
+    pub fn is_inlined_mask(&self) -> Vec<bool> {
+        self.views.iter().map(|v| v.is_inlined()).collect()
+    }
+
+    pub fn hash_all_parallel(&self) -> Vec<u64> {
+        let pool = &self.pool;
+        self.views
+            .par_iter()
+            .map(|v| v.compute_hash(pool))
+            .collect()
+    }
+
+    pub fn filter_by_length(&self, min_len: u32, max_len: u32) -> Self {
+        let mut new_views = Vec::new();
+        for v in &self.views {
+            let l = v.len;
+            if l >= min_len && (max_len == 0 || l <= max_len) {
+                new_views.push(*v);
+            }
+        }
+        Self {
+            views: new_views,
+            pool: self.pool.clone(),
+        }
+    }
+
+    pub fn filter_by_prefix(&self, prefix: &str) -> Self {
+        let prefix_bytes = prefix.as_bytes();
+        let pool = &self.pool;
+        let mut new_views = Vec::new();
+
+        for v in &self.views {
+            let l = v.len as usize;
+            if l < prefix_bytes.len() {
+                continue;
+            }
+            // Fast prefix check
+            if prefix_bytes.len() <= 4 {
+                if v.prefix_or_inline[..prefix_bytes.len()] == prefix_bytes[..] {
+                    new_views.push(*v);
+                }
+            } else if v.prefix_or_inline[..4] == prefix_bytes[..4] {
+                let s = v.as_str(pool);
+                if s.as_bytes().starts_with(prefix_bytes) {
+                    new_views.push(*v);
+                }
+            }
+        }
+
+        Self {
+            views: new_views,
+            pool: self.pool.clone(),
+        }
+    }
+}

data/src/tests.rs

@@ -1,384 +1,384 @@
-use std::fs;
-use std::path::PathBuf;
-
-use crate::coord::StreamCoordinate;
-use crate::dedup::TwoStageDeduplicator;
-use crate::metadata::DualMetadataManager;
-use crate::postprocess::{PostProcessConfig, process_batch_parallel, process_single_record};
-use crate::serialize::{
-    DataValue, build_arrow_record_batch, classify_string_or_json,
-    serialize_columns_to_arrow_ipc_bytes, serialize_columns_to_jsonl_bytes,
-    serialize_columns_to_parquet_bytes,
-};
-use crate::simd_cpu::{contains_byte_fast, detect_cpu_isa, find_newline_fast};
-
-fn temp_test_path(name: &str, ext: &str) -> PathBuf {
-    let mut p = std::env::temp_dir();
-    p.push(format!(
-        "luminamoon_test_{}_{}.{}",
-        name,
-        std::process::id(),
-        ext
-    ));
-    p
-}
-
-#[test]
-fn test_stream_coordinate_roundtrip() {
-    let c = StreamCoordinate {
-        shard_id: 42,
-        byte_offset: 1024 * 1024,
-        row_group_idx: 3,
-        page_idx: 7,
-        row_offset_in_page: 128,
-        global_row_idx: 99999,
-    };
-    let bytes = c.to_bytes();
-    assert_eq!(bytes.len(), StreamCoordinate::BYTE_LEN);
-    let decoded = StreamCoordinate::from_bytes(&bytes);
-    assert_eq!(c, decoded);
-}
-
-#[test]
-fn test_dual_metadata_ping_pong_and_dedup_wal() {
-    let meta_path = temp_test_path("ping_pong_meta", "meta");
-    let _ = fs::remove_file(&meta_path);
-    let _ = fs::remove_file(format!("{}.dedup", meta_path.display()));
-
-    {
-        let mut mgr = DualMetadataManager::open(&meta_path).unwrap();
-        assert_eq!(mgr.committed_batch_id(), 0);
-
-        assert!(mgr.deduplicator.try_stage(1, b"line_alpha").is_some());
-        assert!(mgr.deduplicator.try_stage(1, b"line_alpha").is_none());
-
-        let c1 = StreamCoordinate {
-            shard_id: 1,
-            byte_offset: 512,
-            row_group_idx: 0,
-            page_idx: 0,
-            row_offset_in_page: 0,
-            global_row_idx: 10,
-        };
-        mgr.stage_batch_in_ram(1, c1);
-        let committed = mgr.commit_gpu_batch(1).unwrap();
-        assert_eq!(committed, c1);
-        assert_eq!(mgr.committed_batch_id(), 1);
-    }
-
-    // Re-open and verify recovery
-    {
-        let mut mgr = DualMetadataManager::open(&meta_path).unwrap();
-        assert_eq!(mgr.committed_batch_id(), 1);
-        assert_eq!(mgr.committed_coordinate().shard_id, 1);
-        assert_eq!(mgr.committed_coordinate().byte_offset, 512);
-
-        // "line_alpha" must still be marked as committed
-        assert!(mgr.deduplicator.try_stage(2, b"line_alpha").is_none());
-        assert!(mgr.deduplicator.try_stage(2, b"line_beta").is_some());
-
-        let c2 = StreamCoordinate {
-            shard_id: 1,
-            byte_offset: 1024,
-            row_group_idx: 0,
-            page_idx: 1,
-            row_offset_in_page: 0,
-            global_row_idx: 20,
-        };
-        mgr.stage_batch_in_ram(2, c2);
-        mgr.commit_gpu_batch(2).unwrap();
-    }
-
-    let _ = fs::remove_file(&meta_path);
-    let _ = fs::remove_file(format!("{}.dedup", meta_path.display()));
-}
-
-#[test]
-fn test_two_stage_deduplicator() {
-    let mut dedup = TwoStageDeduplicator::new();
-    assert_eq!(dedup.committed_len(), 0);
-
-    assert!(dedup.try_stage(1, b"apple").is_some());
-    assert!(dedup.try_stage(1, b"apple").is_none());
-    assert!(dedup.try_stage(1, b"banana").is_some());
-    assert_eq!(dedup.staged_len(), 2);
-
-    let committed = dedup.commit_batch(1);
-    assert_eq!(committed.len(), 2);
-    assert_eq!(dedup.committed_len(), 2);
-    assert_eq!(dedup.staged_len(), 0);
-
-    // Staging the same in batch 2 should be rejected
-    assert!(dedup.try_stage(2, b"apple").is_none());
-    assert!(dedup.try_stage(2, b"orange").is_some());
-
-    // Rollback batch 2
-    let rolled = dedup.rollback_batch(2);
-    assert_eq!(rolled, 1);
-    assert!(dedup.try_stage(3, b"orange").is_some());
-}
-
-#[test]
-fn test_simd_cpu_detection_and_scans() {
-    let isa = detect_cpu_isa();
-    assert!(!isa.as_str().is_empty());
-
-    let text = b"hello\r\nworld\nfrom\rsimd";
-    assert!(contains_byte_fast(text, b'\r'));
-    assert!(contains_byte_fast(text, b'\n'));
-    assert!(!contains_byte_fast(text, b'z'));
-
-    let pos = find_newline_fast(text);
-    assert!(pos.is_some());
-}
-
-#[test]
-fn test_postprocess_cleaning_and_json_extraction() {
-    let cfg = PostProcessConfig {
-        trim_whitespace: true,
-        normalize_newlines: true,
-        min_byte_len: 3,
-        max_byte_len: 20,
-        truncate_over_max: true,
-        json_extract_key: Some("data.text".to_owned()),
-    };
-
-    let raw_json = r#"{"data": {"text": "  Hello\r\nWorld!   "}}"#;
-    let cleaned = process_single_record(raw_json, &cfg);
-    assert_eq!(cleaned, Some("Hello\nWorld!".to_owned()));
-
-    // Truncation test
-    let long_json = r#"{"data": {"text": "A very long sentence that will exceed twenty bytes"}}"#;
-    let truncated = process_single_record(long_json, &cfg).unwrap();
-    assert!(truncated.len() <= 20);
-
-    // Parallel batch test
-    let batch = vec![
-        r#"{"data": {"text": "row 1\r\n"}}"#.to_owned(),
-        r#"{"data": {"text": "row 2"}}"#.to_owned(),
-        r#"{"data": {"text": ""}}"#.to_owned(), // filtered due to min_byte_len
-    ];
-    let batch_cleaned = process_batch_parallel(&batch, &cfg);
-    assert_eq!(batch_cleaned.len(), 2);
-    assert_eq!(batch_cleaned[0], "row 1");
-    assert_eq!(batch_cleaned[1], "row 2");
-}
-
-#[test]
-fn test_classify_string_or_json() {
-    let s1 = "just plain text";
-    assert_eq!(classify_string_or_json(s1), DataValue::Text(s1.to_owned()));
-
-    let s2 = r#"{"key": "value", "id": 123}"#;
-    assert_eq!(classify_string_or_json(s2), DataValue::Json(s2.to_owned()));
-}
-
-#[test]
-fn test_serialization_parquet_and_jsonl() {
-    let columns = vec![
-        (
-            "text".to_owned(),
-            vec![
-                DataValue::Text("first".to_owned()),
-                DataValue::Text("second".to_owned()),
-            ],
-        ),
-        (
-            "score".to_owned(),
-            vec![DataValue::Float(0.95), DataValue::Float(0.42)],
-        ),
-        (
-            "label".to_owned(),
-            vec![DataValue::Int(1), DataValue::Int(0)],
-        ),
-    ];
-
-    let (schema, batch) = build_arrow_record_batch(&columns, false).unwrap();
-    assert_eq!(batch.num_rows(), 2);
-    assert_eq!(schema.fields().len(), 3);
-
-    // Parquet serialization
-    let pq_bytes = serialize_columns_to_parquet_bytes(&columns, false).unwrap();
-    assert!(!pq_bytes.is_empty());
-    assert_eq!(&pq_bytes[0..4], b"PAR1");
-
-    // Arrow IPC serialization
-    let ipc_bytes = serialize_columns_to_arrow_ipc_bytes(&columns, false).unwrap();
-    assert!(!ipc_bytes.is_empty());
-    assert_eq!(&ipc_bytes[0..6], b"ARROW1");
-
-    // JSONL serialization
-    let jsonl_bytes = serialize_columns_to_jsonl_bytes(&columns).unwrap();
-    assert!(!jsonl_bytes.is_empty());
-    let jsonl_str = String::from_utf8(jsonl_bytes).unwrap();
-    assert!(jsonl_str.contains("\"text\":\"first\""));
-    assert!(jsonl_str.contains("\"score\":0.95"));
-}
-
-#[test]
-fn test_quality_filter() {
-    use crate::quality::{QualityConfig, evaluate_quality};
-
-    let cfg = QualityConfig {
-        min_chars: 20,
-        max_chars: 0,
-        min_words: 3,
-        min_entropy: 2.5,
-        max_repetition_ratio: 0.35,
-        max_symbol_ratio: 0.30,
-    };
-
-    // 1. High quality normal text
-    let good_text =
-        "The quick brown fox jumps over the lazy dog and runs across the wide green meadow.";
-    let res = evaluate_quality(good_text, &cfg);
-    assert!(res.is_passed);
-    assert!(res.entropy > 2.5);
-    assert!(res.repetition_ratio < 0.2);
-    assert!(res.symbol_ratio < 0.1);
-
-    // 2. Degenerate repetition
-    let repeated =
-        "spam spam spam spam spam spam spam spam spam spam spam spam spam spam spam spam";
-    let res_rep = evaluate_quality(repeated, &cfg);
-    assert!(!res_rep.is_passed);
-    assert!(res_rep.repetition_ratio > 0.35);
-
-    // 3. Code / symbol dump
-    let symbol_dump = "{{{[(((@@@###$$$%%%^^^&&&***)))}}}];;;===+++///<<<>>>!!!???";
-    let res_sym = evaluate_quality(symbol_dump, &cfg);
-    assert!(!res_sym.is_passed);
-}
-
-#[test]
-fn test_minhash_fuzzy_deduplicator() {
-    use crate::minhash::MinHashDeduplicator;
-
-    let mut dedup = MinHashDeduplicator::new(0.80, 4, 64);
-
-    let doc_a =
-        "Artificial intelligence and deep learning models require efficient data pipelines.";
-    // Near duplicate (just 1 typo/word difference)
-    let doc_a_near =
-        "Artificial intelligence and deep learning models require efficient data pipelining.";
-    // Completely different document
-    let doc_b = "Quantum mechanical systems exhibit wave particle duality in atomic physics.";
-
-    assert!(dedup.is_unique(doc_a));
-    // doc_a_near should be caught as near-duplicate (> 80% similarity)
-    assert!(!dedup.is_unique(doc_a_near));
-    // doc_b is different, should pass
-    assert!(dedup.is_unique(doc_b));
-    assert_eq!(dedup.count(), 2);
-}
-
-#[test]
-fn test_string_tensor_cpu_and_gpu() {
-    use crate::string_tensor::StringTensor;
-
-    let raw = vec![
-        "apple",                                        // 5B <= 12B (inlined)
-        "cat",                                          // 3B <= 12B (inlined)
-        "twelve_bytes",                                 // 12B <= 12B (inlined boundary)
-        "thirteen_bytes!",                              // 15B > 12B (referenced into pool)
-        "The quick brown fox jumps over the lazy dog.", // 44B > 12B
-    ];
-
-    let st = StringTensor::from_strings(&raw);
-    assert_eq!(st.len(), 5);
-
-    // Verify inlining
-    assert!(st.views[0].is_inlined());
-    assert!(st.views[1].is_inlined());
-    assert!(st.views[2].is_inlined());
-    assert!(!st.views[3].is_inlined());
-    assert!(!st.views[4].is_inlined());
-
-    // Verify string content recovery
-    assert_eq!(st.get(0).unwrap(), "apple");
-    assert_eq!(st.get(1).unwrap(), "cat");
-    assert_eq!(st.get(2).unwrap(), "twelve_bytes");
-    assert_eq!(st.get(3).unwrap(), "thirteen_bytes!");
-    assert_eq!(
-        st.get(4).unwrap(),
-        "The quick brown fox jumps over the lazy dog."
-    );
-
-    // Fast equality check
-    let v0 = st.views[0];
-    let v1 = st.views[1];
-    assert!(v0.fast_eq(&v0, &st.pool));
-    assert!(!v0.fast_eq(&v1, &st.pool));
-
-    // Convert to GPU buffers: (offsets, payload)
-    let (offsets, payload) = st.to_gpu_buffers();
-    assert_eq!(offsets.len(), 6);
-    assert_eq!(offsets[0], 0);
-    assert_eq!(offsets[5] as usize, payload.len());
-
-    // Reconstruct from GPU buffers
-    let st_gpu = StringTensor::from_gpu_buffers(&offsets, &payload);
-    assert_eq!(st_gpu.len(), 5);
-    assert_eq!(st_gpu.to_vec(), raw);
-
-    // Filtering
-    let long_only = st.filter_by_length(13, 0);
-    assert_eq!(long_only.len(), 2);
-    assert_eq!(long_only.get(0).unwrap(), "thirteen_bytes!");
-
-    let prefix_filtered = st.filter_by_prefix("thirt");
-    assert_eq!(prefix_filtered.len(), 1);
-    assert_eq!(prefix_filtered.get(0).unwrap(), "thirteen_bytes!");
-
-    // Parallel hashing
-    let hashes = st.hash_all_parallel();
-    assert_eq!(hashes.len(), 5);
-    assert!(hashes[0] != hashes[1]);
-}
-
-#[test]
-fn test_fused_pipeline_core() {
-    let raw = vec![
-        "  Valid prose sentence for LLM training alpha\r\n".to_string(),
-        "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa".to_string(), // degenerate repetition -> reject
-        "Valid prose sentence for LLM training beta".to_string(),
-        "Valid prose sentence for LLM training alpha\r\n".to_string(), // duplicate -> reject
-        "".to_string(),                                                // empty -> reject
-    ];
-
-    let post_cfg = PostProcessConfig {
-        min_byte_len: 1,
-        ..Default::default()
-    };
-    let qual_cfg = crate::quality::QualityConfig {
-        min_entropy: 1.5,
-        max_repetition_ratio: 0.4,
-        ..Default::default()
-    };
-    let mut dedup = TwoStageDeduplicator::new();
-
-    let mut survivors = Vec::new();
-    for item in raw {
-        if let Some(cleaned) = process_single_record(&item, &post_cfg) {
-            let eval = crate::quality::evaluate_quality(&cleaned, &qual_cfg);
-            if eval.is_passed && dedup.try_stage(1, cleaned.as_bytes()).is_some() {
-                survivors.push(cleaned);
-            }
-        }
-    }
-
-    assert_eq!(survivors.len(), 2);
-    assert_eq!(survivors[0], "Valid prose sentence for LLM training alpha");
-    assert_eq!(survivors[1], "Valid prose sentence for LLM training beta");
-
-    let st = crate::string_tensor::StringTensor::from_strings(survivors);
-    assert_eq!(st.len(), 2);
-    assert_eq!(
-        st.get(0).unwrap(),
-        "Valid prose sentence for LLM training alpha"
-    );
-    assert_eq!(
-        st.get(1).unwrap(),
-        "Valid prose sentence for LLM training beta"
-    );
-}
+use std::fs;
+use std::path::PathBuf;
+
+use crate::coord::StreamCoordinate;
+use crate::dedup::TwoStageDeduplicator;
+use crate::metadata::DualMetadataManager;
+use crate::postprocess::{PostProcessConfig, process_batch_parallel, process_single_record};
+use crate::serialize::{
+    DataValue, build_arrow_record_batch, classify_string_or_json,
+    serialize_columns_to_arrow_ipc_bytes, serialize_columns_to_jsonl_bytes,
+    serialize_columns_to_parquet_bytes,
+};
+use crate::simd_cpu::{contains_byte_fast, detect_cpu_isa, find_newline_fast};
+
+fn temp_test_path(name: &str, ext: &str) -> PathBuf {
+    let mut p = std::env::temp_dir();
+    p.push(format!(
+        "luminamoon_test_{}_{}.{}",
+        name,
+        std::process::id(),
+        ext
+    ));
+    p
+}
+
+#[test]
+fn test_stream_coordinate_roundtrip() {
+    let c = StreamCoordinate {
+        shard_id: 42,
+        byte_offset: 1024 * 1024,
+        row_group_idx: 3,
+        page_idx: 7,
+        row_offset_in_page: 128,
+        global_row_idx: 99999,
+    };
+    let bytes = c.to_bytes();
+    assert_eq!(bytes.len(), StreamCoordinate::BYTE_LEN);
+    let decoded = StreamCoordinate::from_bytes(&bytes);
+    assert_eq!(c, decoded);
+}
+
+#[test]
+fn test_dual_metadata_ping_pong_and_dedup_wal() {
+    let meta_path = temp_test_path("ping_pong_meta", "meta");
+    let _ = fs::remove_file(&meta_path);
+    let _ = fs::remove_file(format!("{}.dedup", meta_path.display()));
+
+    {
+        let mut mgr = DualMetadataManager::open(&meta_path).unwrap();
+        assert_eq!(mgr.committed_batch_id(), 0);
+
+        assert!(mgr.deduplicator.try_stage(1, b"line_alpha").is_some());
+        assert!(mgr.deduplicator.try_stage(1, b"line_alpha").is_none());
+
+        let c1 = StreamCoordinate {
+            shard_id: 1,
+            byte_offset: 512,
+            row_group_idx: 0,
+            page_idx: 0,
+            row_offset_in_page: 0,
+            global_row_idx: 10,
+        };
+        mgr.stage_batch_in_ram(1, c1);
+        let committed = mgr.commit_gpu_batch(1).unwrap();
+        assert_eq!(committed, c1);
+        assert_eq!(mgr.committed_batch_id(), 1);
+    }
+
+    // Re-open and verify recovery
+    {
+        let mut mgr = DualMetadataManager::open(&meta_path).unwrap();
+        assert_eq!(mgr.committed_batch_id(), 1);
+        assert_eq!(mgr.committed_coordinate().shard_id, 1);
+        assert_eq!(mgr.committed_coordinate().byte_offset, 512);
+
+        // "line_alpha" must still be marked as committed
+        assert!(mgr.deduplicator.try_stage(2, b"line_alpha").is_none());
+        assert!(mgr.deduplicator.try_stage(2, b"line_beta").is_some());
+
+        let c2 = StreamCoordinate {
+            shard_id: 1,
+            byte_offset: 1024,
+            row_group_idx: 0,
+            page_idx: 1,
+            row_offset_in_page: 0,
+            global_row_idx: 20,
+        };
+        mgr.stage_batch_in_ram(2, c2);
+        mgr.commit_gpu_batch(2).unwrap();
+    }
+
+    let _ = fs::remove_file(&meta_path);
+    let _ = fs::remove_file(format!("{}.dedup", meta_path.display()));
+}
+
+#[test]
+fn test_two_stage_deduplicator() {
+    let mut dedup = TwoStageDeduplicator::new();
+    assert_eq!(dedup.committed_len(), 0);
+
+    assert!(dedup.try_stage(1, b"apple").is_some());
+    assert!(dedup.try_stage(1, b"apple").is_none());
+    assert!(dedup.try_stage(1, b"banana").is_some());
+    assert_eq!(dedup.staged_len(), 2);
+
+    let committed = dedup.commit_batch(1);
+    assert_eq!(committed.len(), 2);
+    assert_eq!(dedup.committed_len(), 2);
+    assert_eq!(dedup.staged_len(), 0);
+
+    // Staging the same in batch 2 should be rejected
+    assert!(dedup.try_stage(2, b"apple").is_none());
+    assert!(dedup.try_stage(2, b"orange").is_some());
+
+    // Rollback batch 2
+    let rolled = dedup.rollback_batch(2);
+    assert_eq!(rolled, 1);
+    assert!(dedup.try_stage(3, b"orange").is_some());
+}
+
+#[test]
+fn test_simd_cpu_detection_and_scans() {
+    let isa = detect_cpu_isa();
+    assert!(!isa.as_str().is_empty());
+
+    let text = b"hello\r\nworld\nfrom\rsimd";
+    assert!(contains_byte_fast(text, b'\r'));
+    assert!(contains_byte_fast(text, b'\n'));
+    assert!(!contains_byte_fast(text, b'z'));
+
+    let pos = find_newline_fast(text);
+    assert!(pos.is_some());
+}
+
+#[test]
+fn test_postprocess_cleaning_and_json_extraction() {
+    let cfg = PostProcessConfig {
+        trim_whitespace: true,
+        normalize_newlines: true,
+        min_byte_len: 3,
+        max_byte_len: 20,
+        truncate_over_max: true,
+        json_extract_key: Some("data.text".to_owned()),
+    };
+
+    let raw_json = r#"{"data": {"text": "  Hello\r\nWorld!   "}}"#;
+    let cleaned = process_single_record(raw_json, &cfg);
+    assert_eq!(cleaned, Some("Hello\nWorld!".to_owned()));
+
+    // Truncation test
+    let long_json = r#"{"data": {"text": "A very long sentence that will exceed twenty bytes"}}"#;
+    let truncated = process_single_record(long_json, &cfg).unwrap();
+    assert!(truncated.len() <= 20);
+
+    // Parallel batch test
+    let batch = vec![
+        r#"{"data": {"text": "row 1\r\n"}}"#.to_owned(),
+        r#"{"data": {"text": "row 2"}}"#.to_owned(),
+        r#"{"data": {"text": ""}}"#.to_owned(), // filtered due to min_byte_len
+    ];
+    let batch_cleaned = process_batch_parallel(&batch, &cfg);
+    assert_eq!(batch_cleaned.len(), 2);
+    assert_eq!(batch_cleaned[0], "row 1");
+    assert_eq!(batch_cleaned[1], "row 2");
+}
+
+#[test]
+fn test_classify_string_or_json() {
+    let s1 = "just plain text";
+    assert_eq!(classify_string_or_json(s1), DataValue::Text(s1.to_owned()));
+
+    let s2 = r#"{"key": "value", "id": 123}"#;
+    assert_eq!(classify_string_or_json(s2), DataValue::Json(s2.to_owned()));
+}
+
+#[test]
+fn test_serialization_parquet_and_jsonl() {
+    let columns = vec![
+        (
+            "text".to_owned(),
+            vec![
+                DataValue::Text("first".to_owned()),
+                DataValue::Text("second".to_owned()),
+            ],
+        ),
+        (
+            "score".to_owned(),
+            vec![DataValue::Float(0.95), DataValue::Float(0.42)],
+        ),
+        (
+            "label".to_owned(),
+            vec![DataValue::Int(1), DataValue::Int(0)],
+        ),
+    ];
+
+    let (schema, batch) = build_arrow_record_batch(&columns, false).unwrap();
+    assert_eq!(batch.num_rows(), 2);
+    assert_eq!(schema.fields().len(), 3);
+
+    // Parquet serialization
+    let pq_bytes = serialize_columns_to_parquet_bytes(&columns, false).unwrap();
+    assert!(!pq_bytes.is_empty());
+    assert_eq!(&pq_bytes[0..4], b"PAR1");
+
+    // Arrow IPC serialization
+    let ipc_bytes = serialize_columns_to_arrow_ipc_bytes(&columns, false).unwrap();
+    assert!(!ipc_bytes.is_empty());
+    assert_eq!(&ipc_bytes[0..6], b"ARROW1");
+
+    // JSONL serialization
+    let jsonl_bytes = serialize_columns_to_jsonl_bytes(&columns).unwrap();
+    assert!(!jsonl_bytes.is_empty());
+    let jsonl_str = String::from_utf8(jsonl_bytes).unwrap();
+    assert!(jsonl_str.contains("\"text\":\"first\""));
+    assert!(jsonl_str.contains("\"score\":0.95"));
+}
+
+#[test]
+fn test_quality_filter() {
+    use crate::quality::{QualityConfig, evaluate_quality};
+
+    let cfg = QualityConfig {
+        min_chars: 20,
+        max_chars: 0,
+        min_words: 3,
+        min_entropy: 2.5,
+        max_repetition_ratio: 0.35,
+        max_symbol_ratio: 0.30,
+    };
+
+    // 1. High quality normal text
+    let good_text =
+        "The quick brown fox jumps over the lazy dog and runs across the wide green meadow.";
+    let res = evaluate_quality(good_text, &cfg);
+    assert!(res.is_passed);
+    assert!(res.entropy > 2.5);
+    assert!(res.repetition_ratio < 0.2);
+    assert!(res.symbol_ratio < 0.1);
+
+    // 2. Degenerate repetition
+    let repeated =
+        "spam spam spam spam spam spam spam spam spam spam spam spam spam spam spam spam";
+    let res_rep = evaluate_quality(repeated, &cfg);
+    assert!(!res_rep.is_passed);
+    assert!(res_rep.repetition_ratio > 0.35);
+
+    // 3. Code / symbol dump
+    let symbol_dump = "{{{[(((@@@###$$$%%%^^^&&&***)))}}}];;;===+++///<<<>>>!!!???";
+    let res_sym = evaluate_quality(symbol_dump, &cfg);
+    assert!(!res_sym.is_passed);
+}
+
+#[test]
+fn test_minhash_fuzzy_deduplicator() {
+    use crate::minhash::MinHashDeduplicator;
+
+    let mut dedup = MinHashDeduplicator::new(0.80, 4, 64);
+
+    let doc_a =
+        "Artificial intelligence and deep learning models require efficient data pipelines.";
+    // Near duplicate (just 1 typo/word difference)
+    let doc_a_near =
+        "Artificial intelligence and deep learning models require efficient data pipelining.";
+    // Completely different document
+    let doc_b = "Quantum mechanical systems exhibit wave particle duality in atomic physics.";
+
+    assert!(dedup.is_unique(doc_a));
+    // doc_a_near should be caught as near-duplicate (> 80% similarity)
+    assert!(!dedup.is_unique(doc_a_near));
+    // doc_b is different, should pass
+    assert!(dedup.is_unique(doc_b));
+    assert_eq!(dedup.count(), 2);
+}
+
+#[test]
+fn test_string_tensor_cpu_and_gpu() {
+    use crate::string_tensor::StringTensor;
+
+    let raw = vec![
+        "apple",                                        // 5B <= 12B (inlined)
+        "cat",                                          // 3B <= 12B (inlined)
+        "twelve_bytes",                                 // 12B <= 12B (inlined boundary)
+        "thirteen_bytes!",                              // 15B > 12B (referenced into pool)
+        "The quick brown fox jumps over the lazy dog.", // 44B > 12B
+    ];
+
+    let st = StringTensor::from_strings(&raw);
+    assert_eq!(st.len(), 5);
+
+    // Verify inlining
+    assert!(st.views[0].is_inlined());
+    assert!(st.views[1].is_inlined());
+    assert!(st.views[2].is_inlined());
+    assert!(!st.views[3].is_inlined());
+    assert!(!st.views[4].is_inlined());
+
+    // Verify string content recovery
+    assert_eq!(st.get(0).unwrap(), "apple");
+    assert_eq!(st.get(1).unwrap(), "cat");
+    assert_eq!(st.get(2).unwrap(), "twelve_bytes");
+    assert_eq!(st.get(3).unwrap(), "thirteen_bytes!");
+    assert_eq!(
+        st.get(4).unwrap(),
+        "The quick brown fox jumps over the lazy dog."
+    );
+
+    // Fast equality check
+    let v0 = st.views[0];
+    let v1 = st.views[1];
+    assert!(v0.fast_eq(&v0, &st.pool));
+    assert!(!v0.fast_eq(&v1, &st.pool));
+
+    // Convert to GPU buffers: (offsets, payload)
+    let (offsets, payload) = st.to_gpu_buffers();
+    assert_eq!(offsets.len(), 6);
+    assert_eq!(offsets[0], 0);
+    assert_eq!(offsets[5] as usize, payload.len());
+
+    // Reconstruct from GPU buffers
+    let st_gpu = StringTensor::from_gpu_buffers(&offsets, &payload);
+    assert_eq!(st_gpu.len(), 5);
+    assert_eq!(st_gpu.to_vec(), raw);
+
+    // Filtering
+    let long_only = st.filter_by_length(13, 0);
+    assert_eq!(long_only.len(), 2);
+    assert_eq!(long_only.get(0).unwrap(), "thirteen_bytes!");
+
+    let prefix_filtered = st.filter_by_prefix("thirt");
+    assert_eq!(prefix_filtered.len(), 1);
+    assert_eq!(prefix_filtered.get(0).unwrap(), "thirteen_bytes!");
+
+    // Parallel hashing
+    let hashes = st.hash_all_parallel();
+    assert_eq!(hashes.len(), 5);
+    assert!(hashes[0] != hashes[1]);
+}
+
+#[test]
+fn test_fused_pipeline_core() {
+    let raw = vec![
+        "  Valid prose sentence for LLM training alpha\r\n".to_string(),
+        "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa".to_string(), // degenerate repetition -> reject
+        "Valid prose sentence for LLM training beta".to_string(),
+        "Valid prose sentence for LLM training alpha\r\n".to_string(), // duplicate -> reject
+        "".to_string(),                                                // empty -> reject
+    ];
+
+    let post_cfg = PostProcessConfig {
+        min_byte_len: 1,
+        ..Default::default()
+    };
+    let qual_cfg = crate::quality::QualityConfig {
+        min_entropy: 1.5,
+        max_repetition_ratio: 0.4,
+        ..Default::default()
+    };
+    let mut dedup = TwoStageDeduplicator::new();
+
+    let mut survivors = Vec::new();
+    for item in raw {
+        if let Some(cleaned) = process_single_record(&item, &post_cfg) {
+            let eval = crate::quality::evaluate_quality(&cleaned, &qual_cfg);
+            if eval.is_passed && dedup.try_stage(1, cleaned.as_bytes()).is_some() {
+                survivors.push(cleaned);
+            }
+        }
+    }
+
+    assert_eq!(survivors.len(), 2);
+    assert_eq!(survivors[0], "Valid prose sentence for LLM training alpha");
+    assert_eq!(survivors[1], "Valid prose sentence for LLM training beta");
+
+    let st = crate::string_tensor::StringTensor::from_strings(survivors);
+    assert_eq!(st.len(), 2);
+    assert_eq!(
+        st.get(0).unwrap(),
+        "Valid prose sentence for LLM training alpha"
+    );
+    assert_eq!(
+        st.get(1).unwrap(),
+        "Valid prose sentence for LLM training beta"
+    );
+}

docs/README.md

@@ -1,101 +1,101 @@
-# Loomy (luminamoon) Documentation
-
-<div class="portal-badges">
-  <span class="pill-badge">LuminaV Optimizer v1.6.0</span>
-  <span class="pill-badge pill-badge-green">Data Engine v1.1.0</span>
-  <span class="pill-badge pill-badge-cyan">LSO-Eval Verified</span>
-  <span class="pill-badge pill-badge-cyan">Rust 1.99.1 + Python ABI3T</span>
-</div>
-
-**Loomy (`luminamoon`)** provides low-precision numerical optimization, rational neural network layers, and dataset text preprocessing utilities developed by **Silver Moon ([@cloverxion](https://github.com/cloverxion))** and **Lumina Moon ([@cloverx-id](https://github.com/cloverx-id)) contributors**.
-
-<div class="friendly-grid">
-  <a class="friendly-card" href="en/01_getting_started/">
-    <h3>Getting Started</h3>
-    <p>System requirements, pip install, Rust compilation, and quickstart examples for NumPy, PyTorch, and Data Engine.</p>
-  </a>
-  <a class="friendly-card" href="en/02_user_guide/">
-    <h3>LuminaV Optimizer</h3>
-    <p>Dual-Buffer CLM vs Single-Buffer TD-CLM, mixed-precision, stochastic rounding, and FSDP distributed training.</p>
-  </a>
-  <a class="friendly-card" href="dataloader/standalone_guide/">
-    <h3>Data Engine</h3>
-    <p>Fused SIMD text preprocessing, StringTensor CPU/GPU buffers, deduplication, and 128B Ping-Pong checkpoint recovery.</p>
-  </a>
-  <a class="friendly-card" href="lso_eval/">
-    <h3>LSO-Eval</h3>
-    <p>Z3 SMT safety checks, 2D trajectory benchmarks, 1-Pass EMA parity, and Qwen3.5-4B & Vision Two-Class evaluation.</p>
-  </a>
-  <a class="friendly-card" href="en/03_configuration_api/">
-    <h3>API Reference</h3>
-    <p>Complete signatures, parameter reference tables, validation constraints, and return types for all modules.</p>
-  </a>
-  <a class="friendly-card" href="security/">
-    <h3>Security & Notes</h3>
-    <p>Direct vulnerability disclosure policy to Silver Moon, CodeQL scanning, and third-party library compatibility notes.</p>
-  </a>
-  <a class="friendly-card" href="changelogs/">
-    <h3>Changelogs</h3>
-    <p>Version-by-version release history for Software package distributions and LuminaV algorithm specifications.</p>
-  </a>
-</div>
-
----
-
-## 1. User Documentation
-
-Source directory: `docs/en/`
-
-- **[01. Getting Started](en/01_getting_started.md)**: System requirements, installation, source compilation via Maturin/Rust, hardware & SIMD ISA detection (`LuminaV.detect_hardware()`, `lm.data_cpu_isa()`), and quickstarts for NumPy, PyTorch, and Data Engine (`FusedPipeline`, `StringTensor`, `StateTracker`, `ResumablePipeline`).
-- **[02. User Guide](en/02_user_guide.md)**: Optimizer buffer modes (2-Buffer CLM, 1-Buffer TD-CLM, and Triangle Family T1/T2), mixed-precision & stochastic rounding, step bounding, FSDP distributed training, plus Data Engine SIMD text cleaning, quality filtering, two-stage XXH3-64 & MinHash deduplication, PII masking, StringTensor buffers, and Ping-Pong checkpoint recovery.
-- **[03. Configuration & API Reference](en/03_configuration_api.md)**: Complete parameter reference tables, public methods, and validation rules for `LuminaV`, `TextCleaner`, `QualityFilter`, `StreamDeduplicator`, `MinHashDeduplicator`, `PIIMasker`, `StateTracker`, `StreamCoordinate`, `StringTensor`, `FusedPipeline`, and `ResumablePipeline`.
-- **[04. Architecture & Hardware Acceleration](en/04_architecture_and_hardware.md)**: C4 dispatch hierarchy, Optimizer Rust crates (`core`, `simd`, `vulkan`, `dispatch`, `triton-aot`), Data Engine Rust modules (`constants`, `coord`, `dedup`, `error`, `metadata`, `minhash`, `postprocess`, `py_bridge`, `quality`, `serialize`, `simd_cpu`, `string_tensor`), buffer memory protocols, and free-threaded Python (`abi3t` / PEP 703).
-
----
-
-## 2. Developer Modular Documentation
-
-Technical documentation for contributors working on the Rust crates, GPU kernels, and Python bindings (`docs/optimize/` & `data/docs/`):
-
-### A. LuminaV Optimizer
-Source directories: `docs/optimize/` & `optimizer/`
-
-- **[LuminaV Optimizer Overview & User Guide (`optimizer/README.md`)](optimize/standalone_guide.md)**
-- **[Developer Optimizer Index (`docs/optimize/README.md`)](optimize/README.md)**
-- **[01. Core & Math](optimize/core/README.md)**: CLM, TD-CLM Triangle Defect, PRNG stochastic rounding, and bias correction.
-- **[02. SIMD Vectorization](optimize/simd/README.md)**: AVX-512 and AVX2/FMA x86_64 intrinsics and Rayon parallel threadpool.
-- **[03. Vulkan Compute](optimize/vulkan/README.md)**: Ash Vulkan 1.3/1.4 SPIR-V compute shaders and UMA host buffers.
-- **[04. Triton AOT & GPU Drivers](optimize/triton_aot/README.md)**: AOT compilation, CUDA/HIP driver dynamic loading, and standalone C-API.
-- **[05. Dispatch & Routing](optimize/dispatch/README.md)**: Hardware capability detector and 16,384-element dispatch threshold heuristic.
-- **[06. Python FFI & PyO3 Bridge](optimize/python_ffi/README.md)**: PyO3 GIL release (`py.detach()`), pointer aliasing checks, and NumPy buffer protocols.
-- **[07. Python Runtime & GPU Mixin](optimize/python_runtime/README.md)**: `LuminaV` class, `constants.py`, Triton JIT kernels, FSDP synchronization, and state checkpointing.
-
-### B. LuminaMoon Data Engine
-Source directories: `data/` & `data/docs/`
-
-- **[LuminaMoon Data Engine Overview & User Guide (`data/README.md`)](dataloader/standalone_guide.md)**
-- **[Developer Data Engine Index (`data/docs/README.md`)](dataloader/README.md)**
-- **[01. TextCleaner & QualityFilter](dataloader/simd_and_postprocess/README.md)**: AVX-512/AVX2/NEON SIMD carriage-return stripping, JSON extraction, Shannon entropy, repetition loop, and symbol ratio filters.
-- **[02. StreamDeduplicator & MinHashDeduplicator](dataloader/dedup/README.md)**: Exact streaming deduplication with `xxh3_64` (`IdentityBuildHasher` SwissTable), WAL persistence, and 64-permutation MinHash LSH fuzzy deduplication.
-- **[03. PIIMasker & Data Sanitization](dataloader/simd_and_postprocess/README.md)**: Pattern-based and regex masking for emails, IPv4 addresses, phone numbers, and secret tokens.
-- **[04. StateTracker & StreamCoordinate](dataloader/coord_and_metadata/README.md)**: 32-byte `StreamCoordinate` navigation, dual-slot 128-byte Ping-Pong checkpoint persistence, and multi-epoch state tracking.
-- **[05. StringTensor & Memory Layouts](dataloader/coord_and_metadata/README.md)**: CPU StringView (inlined <= 12 bytes) and GPU contiguous offset/payload buffers compatible with PyTorch, NumPy, Arrow, and Polars.
-- **[06. FusedPipeline & ResumablePipeline](dataloader/python_ffi_and_runtime/README.md)**: Multi-threaded Rayon single-pass preprocessing pipeline and resumable batch pipeline with background prefetcher.
-- **[07. Multimodal Serialization](dataloader/multimodal_and_hf_hub/README.md)**: Serialization to JSONL, Parquet with Zstd compression and embedded image payloads, Arrow IPC, and CSV.
-
-### C. Ra-Family Neural Architecture & Theoretical References
-Source directories: `docs/nn/` & `references/`
-
-- **[Ra-Family Overview (`docs/nn/README.md`)](nn/README.md)**: Square-root-free `RaNorm` & `RaBatchNorm2d`, rational `RaTanh` & `RaGELU`, bounded `PosRaGLU` / `RaGLU` / `TriGLU`, and compact-support `ra_softmax` & `RaAttention`.
-- **[Ra-Family Standalone Guide (`docs/nn/standalone_guide.md`)](nn/standalone_guide.md)**: Drop-in Transformer block replacement, `RaBatchNorm2d` `Conv2d` parameter folding, and backend introspection.
-- **[Theoretical References (`docs/references.md`)](references.md)**: Component mapping, literature comparisons across all 29 referenced papers, and BibTeX database (`references/bibliography.bib`).
-
----
-
-## 3. LSO-Eval Verification & Library Compatibility Notes
-
-Source directories: `eval/`, `docs/library_audit/`, & `data/docs/library_audit/`
-
-- **[LSO-Eval Optimizer Evaluation & Verification (`eval/README.md`)](lso_eval.md)**: Algorithm-agnostic Z3 SMT solver invariant checks, 2D trajectory benchmarks, numerical parity tests, and Two-Class 4B LLM & Vision evaluation (`LuminaV v1.5.3–v1.6.0`).
-- **[Optimizer Library Notes (`docs/library_audit/`)](library_audit/README.md)**: Version and ABI compatibility notes for Rust 1.99.1 toolchain, PyO3/Maturin, PyTorch 2.14.1, Triton JIT 3.8.0, CUDA/ROCm, and Vulkan Ash compute pipeline.
-- **[Data Engine Library Notes (`data/docs/library_audit/`)](library_audit/README.md)**: Version and compatibility notes for Rust 1.99.1 / Edition 2024, `pyo3 0.30.1` (`abi3t`), Apache Arrow/Parquet/IPC `60.0.0`, `rayon 1.12.0`, `xxhash-rust 0.8.19` (`xxh3`), `crc32fast 1.5.2`, `hashbrown 0.17.1`, `parking_lot 0.12.5`, `serde 1.0.229`, and HuggingFace Hub API protocols.
+# Loomy (luminamoon) Documentation
+
+<div class="portal-badges">
+  <span class="pill-badge">LuminaV Optimizer v1.6.0</span>
+  <span class="pill-badge pill-badge-green">Data Engine v1.1.0</span>
+  <span class="pill-badge pill-badge-cyan">LSO-Eval Verified</span>
+  <span class="pill-badge pill-badge-cyan">Rust 1.99.1 + Python ABI3T</span>
+</div>
+
+**Loomy (`luminamoon`)** provides low-precision numerical optimization, rational neural network layers, and dataset text preprocessing utilities developed by **Silver Moon ([@cloverxion](https://github.com/cloverxion))** and **Lumina Moon ([@cloverx-id](https://github.com/cloverx-id)) contributors**.
+
+<div class="friendly-grid">
+  <a class="friendly-card" href="en/01_getting_started/">
+    <h3>Getting Started</h3>
+    <p>System requirements, pip install, Rust compilation, and quickstart examples for NumPy, PyTorch, and Data Engine.</p>
+  </a>
+  <a class="friendly-card" href="en/02_user_guide/">
+    <h3>LuminaV Optimizer</h3>
+    <p>Dual-Buffer CLM vs Single-Buffer TD-CLM, mixed-precision, stochastic rounding, and FSDP distributed training.</p>
+  </a>
+  <a class="friendly-card" href="dataloader/standalone_guide/">
+    <h3>Data Engine</h3>
+    <p>Fused SIMD text preprocessing, StringTensor CPU/GPU buffers, deduplication, and 128B Ping-Pong checkpoint recovery.</p>
+  </a>
+  <a class="friendly-card" href="lso_eval/">
+    <h3>LSO-Eval</h3>
+    <p>Z3 SMT safety checks, 2D trajectory benchmarks, 1-Pass EMA parity, and Qwen3.5-4B & Vision Two-Class evaluation.</p>
+  </a>
+  <a class="friendly-card" href="en/03_configuration_api/">
+    <h3>API Reference</h3>
+    <p>Complete signatures, parameter reference tables, validation constraints, and return types for all modules.</p>
+  </a>
+  <a class="friendly-card" href="security/">
+    <h3>Security & Notes</h3>
+    <p>Direct vulnerability disclosure policy to Silver Moon, CodeQL scanning, and third-party library compatibility notes.</p>
+  </a>
+  <a class="friendly-card" href="changelogs/">
+    <h3>Changelogs</h3>
+    <p>Version-by-version release history for Software package distributions and LuminaV algorithm specifications.</p>
+  </a>
+</div>
+
+---
+
+## 1. User Documentation
+
+Source directory: `docs/en/`
+
+- **[01. Getting Started](en/01_getting_started.md)**: System requirements, installation, source compilation via Maturin/Rust, hardware & SIMD ISA detection (`LuminaV.detect_hardware()`, `lm.data_cpu_isa()`), and quickstarts for NumPy, PyTorch, and Data Engine (`FusedPipeline`, `StringTensor`, `StateTracker`, `ResumablePipeline`).
+- **[02. User Guide](en/02_user_guide.md)**: Optimizer buffer modes (2-Buffer CLM, 1-Buffer TD-CLM, and Triangle Family T1/T2), mixed-precision & stochastic rounding, step bounding, FSDP distributed training, plus Data Engine SIMD text cleaning, quality filtering, two-stage XXH3-64 & MinHash deduplication, PII masking, StringTensor buffers, and Ping-Pong checkpoint recovery.
+- **[03. Configuration & API Reference](en/03_configuration_api.md)**: Complete parameter reference tables, public methods, and validation rules for `LuminaV`, `TextCleaner`, `QualityFilter`, `StreamDeduplicator`, `MinHashDeduplicator`, `PIIMasker`, `StateTracker`, `StreamCoordinate`, `StringTensor`, `FusedPipeline`, and `ResumablePipeline`.
+- **[04. Architecture & Hardware Acceleration](en/04_architecture_and_hardware.md)**: C4 dispatch hierarchy, Optimizer Rust crates (`core`, `simd`, `vulkan`, `dispatch`, `triton-aot`), Data Engine Rust modules (`constants`, `coord`, `dedup`, `error`, `metadata`, `minhash`, `postprocess`, `py_bridge`, `quality`, `serialize`, `simd_cpu`, `string_tensor`), buffer memory protocols, and free-threaded Python (`abi3t` / PEP 703).
+
+---
+
+## 2. Developer Modular Documentation
+
+Technical documentation for contributors working on the Rust crates, GPU kernels, and Python bindings (`docs/optimize/` & `data/docs/`):
+
+### A. LuminaV Optimizer
+Source directories: `docs/optimize/` & `optimizer/`
+
+- **[LuminaV Optimizer Overview & User Guide (`optimizer/README.md`)](optimize/standalone_guide.md)**
+- **[Developer Optimizer Index (`docs/optimize/README.md`)](optimize/README.md)**
+- **[01. Core & Math](optimize/core/README.md)**: CLM, TD-CLM Triangle Defect, PRNG stochastic rounding, and bias correction.
+- **[02. SIMD Vectorization](optimize/simd/README.md)**: AVX-512 and AVX2/FMA x86_64 intrinsics and Rayon parallel threadpool.
+- **[03. Vulkan Compute](optimize/vulkan/README.md)**: Ash Vulkan 1.3/1.4 SPIR-V compute shaders and UMA host buffers.
+- **[04. Triton AOT & GPU Drivers](optimize/triton_aot/README.md)**: AOT compilation, CUDA/HIP driver dynamic loading, and standalone C-API.
+- **[05. Dispatch & Routing](optimize/dispatch/README.md)**: Hardware capability detector and 16,384-element dispatch threshold heuristic.
+- **[06. Python FFI & PyO3 Bridge](optimize/python_ffi/README.md)**: PyO3 GIL release (`py.detach()`), pointer aliasing checks, and NumPy buffer protocols.
+- **[07. Python Runtime & GPU Mixin](optimize/python_runtime/README.md)**: `LuminaV` class, `constants.py`, Triton JIT kernels, FSDP synchronization, and state checkpointing.
+
+### B. LuminaMoon Data Engine
+Source directories: `data/` & `data/docs/`
+
+- **[LuminaMoon Data Engine Overview & User Guide (`data/README.md`)](dataloader/standalone_guide.md)**
+- **[Developer Data Engine Index (`data/docs/README.md`)](dataloader/README.md)**
+- **[01. TextCleaner & QualityFilter](dataloader/simd_and_postprocess/README.md)**: AVX-512/AVX2/NEON SIMD carriage-return stripping, JSON extraction, Shannon entropy, repetition loop, and symbol ratio filters.
+- **[02. StreamDeduplicator & MinHashDeduplicator](dataloader/dedup/README.md)**: Exact streaming deduplication with `xxh3_64` (`IdentityBuildHasher` SwissTable), WAL persistence, and 64-permutation MinHash LSH fuzzy deduplication.
+- **[03. PIIMasker & Data Sanitization](dataloader/simd_and_postprocess/README.md)**: Pattern-based and regex masking for emails, IPv4 addresses, phone numbers, and secret tokens.
+- **[04. StateTracker & StreamCoordinate](dataloader/coord_and_metadata/README.md)**: 32-byte `StreamCoordinate` navigation, dual-slot 128-byte Ping-Pong checkpoint persistence, and multi-epoch state tracking.
+- **[05. StringTensor & Memory Layouts](dataloader/coord_and_metadata/README.md)**: CPU StringView (inlined <= 12 bytes) and GPU contiguous offset/payload buffers compatible with PyTorch, NumPy, Arrow, and Polars.
+- **[06. FusedPipeline & ResumablePipeline](dataloader/python_ffi_and_runtime/README.md)**: Multi-threaded Rayon single-pass preprocessing pipeline and resumable batch pipeline with background prefetcher.
+- **[07. Multimodal Serialization](dataloader/multimodal_and_hf_hub/README.md)**: Serialization to JSONL, Parquet with Zstd compression and embedded image payloads, Arrow IPC, and CSV.
+
+### C. Ra-Family Neural Architecture & Theoretical References
+Source directories: `docs/nn/` & `references/`
+
+- **[Ra-Family Overview (`docs/nn/README.md`)](nn/README.md)**: Square-root-free `RaNorm` & `RaBatchNorm2d`, rational `RaTanh` & `RaGELU`, bounded `PosRaGLU` / `RaGLU` / `TriGLU`, and compact-support `ra_softmax` & `RaAttention`.
+- **[Ra-Family Standalone Guide (`docs/nn/standalone_guide.md`)](nn/standalone_guide.md)**: Drop-in Transformer block replacement, `RaBatchNorm2d` `Conv2d` parameter folding, and backend introspection.
+- **[Theoretical References (`docs/references.md`)](references.md)**: Component mapping, literature comparisons across all 29 referenced papers, and BibTeX database (`references/bibliography.bib`).
+
+---
+
+## 3. LSO-Eval Verification & Library Compatibility Notes
+
+Source directories: `eval/`, `docs/library_audit/`, & `data/docs/library_audit/`
+
+- **[LSO-Eval Optimizer Evaluation & Verification (`eval/README.md`)](lso_eval.md)**: Algorithm-agnostic Z3 SMT solver invariant checks, 2D trajectory benchmarks, numerical parity tests, and Two-Class 4B LLM & Vision evaluation (`LuminaV v1.5.3–v1.6.0`).
+- **[Optimizer Library Notes (`docs/library_audit/`)](library_audit/README.md)**: Version and ABI compatibility notes for Rust 1.99.1 toolchain, PyO3/Maturin, PyTorch 2.14.1, Triton JIT 3.8.0, CUDA/ROCm, and Vulkan Ash compute pipeline.
+- **[Data Engine Library Notes (`data/docs/library_audit/`)](library_audit/README.md)**: Version and compatibility notes for Rust 1.99.1 / Edition 2024, `pyo3 0.30.1` (`abi3t`), Apache Arrow/Parquet/IPC `60.0.0`, `rayon 1.12.0`, `xxhash-rust 0.8.19` (`xxh3`), `crc32fast 1.5.2`, `hashbrown 0.17.1`, `parking_lot 0.12.5`, `serde 1.0.229`, and HuggingFace Hub API protocols.

docs/dataloader/README.md

@@ -1,69 +1,69 @@
-# LuminaMoon Data Engine Architecture
-
-Technical architecture and module reference for the **LuminaMoon Data Engine (`luminamoon.data` / `luminamoon-dataloader`)**.
-
----
-
-## 1. Subsystem Architecture & Modularity
-
-The Data Engine consists of a modular Python interface (`python/luminamoon/data/` with pure-Python `fallbacks/`), a `PyO3 0.30.1` (`abi3t`) FFI bridge (`data/src/py_bridge.rs`), and a Rust SIMD, deduplication, and columnar processing crate (`data/src/`):
-
-```mermaid
-flowchart LR
-    Input["Raw Text / Iterable / HF Stream"] --> Cleaner["1. SIMD TextCleaner & PostProcess (AVX-512BW / AVX2 / NEON)"]
-    Cleaner --> Quality["2. QualityFilter (Shannon Entropy, Repetition & Symbol Ratio)"]
-    Quality --> Dedup["3. TwoStageDeduplicator (XXH3-64 SwissTable) & MinHash LSH"]
-    Dedup --> Masker["4. PIIMasker (Email, IP, API Key, Card)"]
-    Masker --> Tensor["5. StringTensor (16B CPU StringView16 & Contiguous GPU Buffers)"]
-    Tensor --> Export["Arrow / Polars / PyTorch GPU Tensors / Parquet / IPC"]
-    Tensor --> Meta["6. StateTracker (128B Ping-Pong Slot A/B CRC32 & .dedup WAL)"]
-```
-
----
-
-## 2. Directory Navigation & Technical Specs
-
-| Module | Source Path | Focus & Core Invariants | Spec Document |
-| :--- | :--- | :--- | :--- |
-| **Coordinates, Constants & Dual Metadata** | `data/src/constants.rs`, `error.rs`, `coord.rs`, `metadata.rs` | 32-byte `StreamCoordinate`, 128-byte Ping-Pong (`Slot A`/`Slot B`) CRC32 checkpoint, `.dedup` WAL, Damerau-Levenshtein suggestions | [01. Coord & Metadata](coord_and_metadata/README.md) |
-| **Two-Stage XXH3-64 Deduplication** | `data/src/dedup.rs` | `TwoStageDeduplicator`, `xxh3_64`, `IdentityBuildHasher` SwissTable, staged vs committed sets | [02. Deduplication](dedup/README.md) |
-| **SIMD & Parallel Post-Process** | `data/src/simd_cpu.rs`, `postprocess.rs` | `AVX-512BW` / `AVX2` / `NEON` byte kernels, `CacheAlignedBuffer`, Rayon UTF-8/CRLF/JSON dotted-path extraction | [03. SIMD & Post-Process](simd_and_postprocess/README.md) |
-| **Heuristic Quality & MinHash LSH** | `data/src/quality.rs`, `minhash.rs` | Shannon character entropy, n-gram repetition ratio, symbol ratio, and 64-permutation MinHash Jaccard similarity | [04. Quality & MinHash](sources_and_streaming/README.md) |
-| **Dual-Layout `StringTensor` & Adapters** | `data/src/string_tensor.rs`, `python/luminamoon/data/adapters.py` | 16-byte `StringView16` (<=12B inline), GPU contiguous `uint32` offsets + `uint8` payload, `ResumablePipeline` async prefetch | [05. StringTensor & Adapters](preram/README.md) |
-| **Columnar Serialization Engine** | `data/src/serialize.rs`, `python/luminamoon/data/serialization.py` | `DataValue` enum, ZSTD Parquet (`hf_image_struct`), Apache Arrow IPC (`ARROW1`), and NDJSON | [06. Serialization](multimodal_and_hf_hub/README.md) |
-| **Python FFI & Modular Runtime** | `data/src/py_bridge.rs`, `python/luminamoon/data/` | `abi3t` bindings, `FusedPipeline`, `TextCleaner`, `PIIMasker`, `StateTracker`, and `fallbacks/` | [07. Python FFI & Runtime](python_ffi_and_runtime/README.md) |
-
----
-
-## 3. Developer Invariants & Coding Standards
-
-1. **Dual-Layout String Representation (`StringTensor`)**:
-   - CPU operations use a 16-byte `StringView16` struct (storing strings $\le 12$ bytes inline and comparing 4-byte prefixes in a register), while GPU/Arrow exports emit contiguous `uint32` offset and `uint8` UTF-8 payload buffers.
-2. **Dual-Slot Metadata & Two-Stage Deduplication**:
-   - In-flight batches staged via `StateTracker.update()` and `StreamDeduplicator.filter_batch()` update in-memory state first. On-disk state (`Slot A` / `Slot B` protected by IEEE CRC32 and the `.dedup` WAL) is written when `.commit()` is called.
-3. **Self-Contained SIMD Module**:
-   - `luminamoon-dataloader` maintains its own `AVX-512BW`, `AVX2`, `NEON`, and `Scalar` kernels in `simd_cpu.rs` (cached via `OnceLock<CpuIsa>`) independent of the `optimizer` crates.
-4. **GIL Release (`py.detach()`)**:
-   - Multi-threaded Rayon batch cleaning, quality filtering, `xxh3_64` hashing, `FusedPipeline` execution, and Parquet/Arrow IPC/JSONL serialization execute inside `py.detach()` on CPython 3.10–3.15+ (`abi3` and `abi3t`).
-
----
-
-## 4. Local Build & Test Verification
-
-```bash
-# Run static analysis across the Rust workspace
-cargo clippy --workspace --all-targets -- -D warnings
-
-# Build release extension locally
-maturin develop --release
-
-# Run Rust DataLoader unit and integration tests
-cargo test -p luminamoon-dataloader
-
-# Run Python static type check
-npx pyright python/luminamoon
-
-# Run Python Data Engine & StringTensor test suites
-python -B tests/test_data_modular.py
-python -B tests/test_gpu_vs_cpu_strings.py
-```
+# LuminaMoon Data Engine Architecture
+
+Technical architecture and module reference for the **LuminaMoon Data Engine (`luminamoon.data` / `luminamoon-dataloader`)**.
+
+---
+
+## 1. Subsystem Architecture & Modularity
+
+The Data Engine consists of a modular Python interface (`python/luminamoon/data/` with pure-Python `fallbacks/`), a `PyO3 0.30.1` (`abi3t`) FFI bridge (`data/src/py_bridge.rs`), and a Rust SIMD, deduplication, and columnar processing crate (`data/src/`):
+
+```mermaid
+flowchart LR
+    Input["Raw Text / Iterable / HF Stream"] --> Cleaner["1. SIMD TextCleaner & PostProcess (AVX-512BW / AVX2 / NEON)"]
+    Cleaner --> Quality["2. QualityFilter (Shannon Entropy, Repetition & Symbol Ratio)"]
+    Quality --> Dedup["3. TwoStageDeduplicator (XXH3-64 SwissTable) & MinHash LSH"]
+    Dedup --> Masker["4. PIIMasker (Email, IP, API Key, Card)"]
+    Masker --> Tensor["5. StringTensor (16B CPU StringView16 & Contiguous GPU Buffers)"]
+    Tensor --> Export["Arrow / Polars / PyTorch GPU Tensors / Parquet / IPC"]
+    Tensor --> Meta["6. StateTracker (128B Ping-Pong Slot A/B CRC32 & .dedup WAL)"]
+```
+
+---
+
+## 2. Directory Navigation & Technical Specs
+
+| Module | Source Path | Focus & Core Invariants | Spec Document |
+| :--- | :--- | :--- | :--- |
+| **Coordinates, Constants & Dual Metadata** | `data/src/constants.rs`, `error.rs`, `coord.rs`, `metadata.rs` | 32-byte `StreamCoordinate`, 128-byte Ping-Pong (`Slot A`/`Slot B`) CRC32 checkpoint, `.dedup` WAL, Damerau-Levenshtein suggestions | [01. Coord & Metadata](coord_and_metadata/README.md) |
+| **Two-Stage XXH3-64 Deduplication** | `data/src/dedup.rs` | `TwoStageDeduplicator`, `xxh3_64`, `IdentityBuildHasher` SwissTable, staged vs committed sets | [02. Deduplication](dedup/README.md) |
+| **SIMD & Parallel Post-Process** | `data/src/simd_cpu.rs`, `postprocess.rs` | `AVX-512BW` / `AVX2` / `NEON` byte kernels, `CacheAlignedBuffer`, Rayon UTF-8/CRLF/JSON dotted-path extraction | [03. SIMD & Post-Process](simd_and_postprocess/README.md) |
+| **Heuristic Quality & MinHash LSH** | `data/src/quality.rs`, `minhash.rs` | Shannon character entropy, n-gram repetition ratio, symbol ratio, and 64-permutation MinHash Jaccard similarity | [04. Quality & MinHash](sources_and_streaming/README.md) |
+| **Dual-Layout `StringTensor` & Adapters** | `data/src/string_tensor.rs`, `python/luminamoon/data/adapters.py` | 16-byte `StringView16` (<=12B inline), GPU contiguous `uint32` offsets + `uint8` payload, `ResumablePipeline` async prefetch | [05. StringTensor & Adapters](preram/README.md) |
+| **Columnar Serialization Engine** | `data/src/serialize.rs`, `python/luminamoon/data/serialization.py` | `DataValue` enum, ZSTD Parquet (`hf_image_struct`), Apache Arrow IPC (`ARROW1`), and NDJSON | [06. Serialization](multimodal_and_hf_hub/README.md) |
+| **Python FFI & Modular Runtime** | `data/src/py_bridge.rs`, `python/luminamoon/data/` | `abi3t` bindings, `FusedPipeline`, `TextCleaner`, `PIIMasker`, `StateTracker`, and `fallbacks/` | [07. Python FFI & Runtime](python_ffi_and_runtime/README.md) |
+
+---
+
+## 3. Developer Invariants & Coding Standards
+
+1. **Dual-Layout String Representation (`StringTensor`)**:
+   - CPU operations use a 16-byte `StringView16` struct (storing strings $\le 12$ bytes inline and comparing 4-byte prefixes in a register), while GPU/Arrow exports emit contiguous `uint32` offset and `uint8` UTF-8 payload buffers.
+2. **Dual-Slot Metadata & Two-Stage Deduplication**:
+   - In-flight batches staged via `StateTracker.update()` and `StreamDeduplicator.filter_batch()` update in-memory state first. On-disk state (`Slot A` / `Slot B` protected by IEEE CRC32 and the `.dedup` WAL) is written when `.commit()` is called.
+3. **Self-Contained SIMD Module**:
+   - `luminamoon-dataloader` maintains its own `AVX-512BW`, `AVX2`, `NEON`, and `Scalar` kernels in `simd_cpu.rs` (cached via `OnceLock<CpuIsa>`) independent of the `optimizer` crates.
+4. **GIL Release (`py.detach()`)**:
+   - Multi-threaded Rayon batch cleaning, quality filtering, `xxh3_64` hashing, `FusedPipeline` execution, and Parquet/Arrow IPC/JSONL serialization execute inside `py.detach()` on CPython 3.10–3.15+ (`abi3` and `abi3t`).
+
+---
+
+## 4. Local Build & Test Verification
+
+```bash
+# Run static analysis across the Rust workspace
+cargo clippy --workspace --all-targets -- -D warnings
+
+# Build release extension locally
+maturin develop --release
+
+# Run Rust DataLoader unit and integration tests
+cargo test -p luminamoon-dataloader
+
+# Run Python static type check
+npx pyright python/luminamoon
+
+# Run Python Data Engine & StringTensor test suites
+python -B tests/test_data_modular.py
+python -B tests/test_gpu_vs_cpu_strings.py
+```

docs/dataloader/coord_and_metadata/README.md

@@ -1,70 +1,70 @@
-# Stream Coordinates & Ping-Pong Checkpoint Recovery
-
-[`data/src/coord.rs`](https://github.com/cloverx-id/luminamoon/blob/main/data/src/coord.rs) and [`data/src/metadata.rs`](https://github.com/cloverx-id/luminamoon/blob/main/data/src/metadata.rs) define stream coordinate tracking and atomic dual-slot checkpointing, enabling dataset resume without re-reading or re-hashing earlier shards.
-
----
-
-## 1. 32-Byte `StreamCoordinate` Binary Layout (`src/coord.rs`)
-
-Positions across multi-shard streams are addressed by a fixed 32-byte little-endian struct:
-
-| Byte Offset | Field | Rust Type | Width | Description |
-| :--- | :--- | :--- | :--- | :--- |
-| `0..4` | `shard_id` | `u32` | 4 B | Active index in the expanded shard URI list. |
-| `4..12` | `byte_offset` | `u64` | 8 B | Exact byte offset for `.jsonl` / `.txt` `mmap` or HTTP 206 Range streams. |
-| `12..16` | `row_group_idx` | `u32` | 4 B | Active Parquet Row Group index. |
-| `16..20` | `page_idx` | `u32` | 4 B | Reserved page index within the Row Group. |
-| `20..24` | `row_offset_in_page` | `u32` | 4 B | Exact row offset within the active Parquet Row Group (`RowSelector::skip`). |
-| `24..32` | `global_row_idx` | `u64` | 8 B | Cumulative number of raw records scanned across all shards. |
-
-```rust
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
-pub struct StreamCoordinate {
-    pub shard_id: u32,
-    pub byte_offset: u64,
-    pub row_group_idx: u32,
-    pub page_idx: u32,
-    pub row_offset_in_page: u32,
-    pub global_row_idx: u64,
-}
-```
-
----
-
-## 2. 128-Byte Ping-Pong Checkpoint File (`src/metadata.rs`)
-
-To prevent checkpoint corruption from sudden power loss or process interruption, `DualMetadataManager` maintains a fixed 128-byte `.meta` file divided into two 64-byte slots (`Slot A` at `0..64` and `Slot B` at `64..128`):
-
-```mermaid
-flowchart LR
-    Commit["commit_gpu_batch(batch_id)"] --> WAL["Flush Hashes to .dedup WAL"]
-    WAL --> Epoch["Increment Epoch Counter"]
-    Epoch --> SlotA["Odd Epoch: Slot A (Bytes 0..64 + CRC32)"]
-    Epoch --> SlotB["Even Epoch: Slot B (Bytes 64..128 + CRC32)"]
-    SlotA --> Recover["Verify CRC32 & Select max(epoch)"]
-    SlotB --> Recover
-```
-
-### 64-Byte `DiskSlot` Wire Format
-
-| Byte Range | Field | Type | Description |
-| :--- | :--- | :--- | :--- |
-| `0..4` | `SLOT_MAGIC` | `[u8; 4]` | Constant magic header `*b"LMDL"`. |
-| `4..8` | `SLOT_VERSION` | `u32` | Format version (`1`). |
-| `8..16` | `epoch` | `u64` | Monotonically increasing commit sequence number. |
-| `16..24` | `committed_batch_id` | `u64` | Last batch ID confirmed consumed by the GPU training step. |
-| `24..56` | `coord` | `[u8; 32]` | Encoded `StreamCoordinate` immediately following the committed batch. |
-| `56..60` | `dedup_count` | `u32` | Total number of committed 8-byte `xxh3_64` hashes valid in `.meta.dedup`. |
-| `60..64` | `crc32` | `u32` | Hardware-accelerated IEEE CRC32 (`crc32fast`) over bytes `0..60`. |
-
----
-
-## 3. Recovery & Truncation Invariants
-
-When initializing `DualMetadataManager`:
-
-1. Both `Slot A` and `Slot B` are decoded and verified against `SLOT_MAGIC`, `SLOT_VERSION`, and their 32-bit IEEE CRC32 checksum.
-2. If both slots are valid, the slot with the higher `epoch` is selected:
-   $$\text{Slot}_{\text{active}} = \arg\max_{S \in \{A, B\}} S.\text{epoch}$$
-3. If one slot was torn during a system crash, its CRC32 check fails and the manager falls back to the intact partner slot.
-4. The companion `.meta.dedup` write-ahead log is truncated to exactly $\text{dedup}_{\text{count}} \times 8$ bytes (`dedup_file.set_len(valid_dedup_bytes)`), discarding any uncommitted partial hash appends before reconstructing `TwoStageDeduplicator::with_committed`.
+# Stream Coordinates & Ping-Pong Checkpoint Recovery
+
+[`data/src/coord.rs`](https://github.com/cloverx-id/luminamoon/blob/main/data/src/coord.rs) and [`data/src/metadata.rs`](https://github.com/cloverx-id/luminamoon/blob/main/data/src/metadata.rs) define stream coordinate tracking and atomic dual-slot checkpointing, enabling dataset resume without re-reading or re-hashing earlier shards.
+
+---
+
+## 1. 32-Byte `StreamCoordinate` Binary Layout (`src/coord.rs`)
+
+Positions across multi-shard streams are addressed by a fixed 32-byte little-endian struct:
+
+| Byte Offset | Field | Rust Type | Width | Description |
+| :--- | :--- | :--- | :--- | :--- |
+| `0..4` | `shard_id` | `u32` | 4 B | Active index in the expanded shard URI list. |
+| `4..12` | `byte_offset` | `u64` | 8 B | Exact byte offset for `.jsonl` / `.txt` `mmap` or HTTP 206 Range streams. |
+| `12..16` | `row_group_idx` | `u32` | 4 B | Active Parquet Row Group index. |
+| `16..20` | `page_idx` | `u32` | 4 B | Reserved page index within the Row Group. |
+| `20..24` | `row_offset_in_page` | `u32` | 4 B | Exact row offset within the active Parquet Row Group (`RowSelector::skip`). |
+| `24..32` | `global_row_idx` | `u64` | 8 B | Cumulative number of raw records scanned across all shards. |
+
+```rust
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+pub struct StreamCoordinate {
+    pub shard_id: u32,
+    pub byte_offset: u64,
+    pub row_group_idx: u32,
+    pub page_idx: u32,
+    pub row_offset_in_page: u32,
+    pub global_row_idx: u64,
+}
+```
+
+---
+
+## 2. 128-Byte Ping-Pong Checkpoint File (`src/metadata.rs`)
+
+To prevent checkpoint corruption from sudden power loss or process interruption, `DualMetadataManager` maintains a fixed 128-byte `.meta` file divided into two 64-byte slots (`Slot A` at `0..64` and `Slot B` at `64..128`):
+
+```mermaid
+flowchart LR
+    Commit["commit_gpu_batch(batch_id)"] --> WAL["Flush Hashes to .dedup WAL"]
+    WAL --> Epoch["Increment Epoch Counter"]
+    Epoch --> SlotA["Odd Epoch: Slot A (Bytes 0..64 + CRC32)"]
+    Epoch --> SlotB["Even Epoch: Slot B (Bytes 64..128 + CRC32)"]
+    SlotA --> Recover["Verify CRC32 & Select max(epoch)"]
+    SlotB --> Recover
+```
+
+### 64-Byte `DiskSlot` Wire Format
+
+| Byte Range | Field | Type | Description |
+| :--- | :--- | :--- | :--- |
+| `0..4` | `SLOT_MAGIC` | `[u8; 4]` | Constant magic header `*b"LMDL"`. |
+| `4..8` | `SLOT_VERSION` | `u32` | Format version (`1`). |
+| `8..16` | `epoch` | `u64` | Monotonically increasing commit sequence number. |
+| `16..24` | `committed_batch_id` | `u64` | Last batch ID confirmed consumed by the GPU training step. |
+| `24..56` | `coord` | `[u8; 32]` | Encoded `StreamCoordinate` immediately following the committed batch. |
+| `56..60` | `dedup_count` | `u32` | Total number of committed 8-byte `xxh3_64` hashes valid in `.meta.dedup`. |
+| `60..64` | `crc32` | `u32` | Hardware-accelerated IEEE CRC32 (`crc32fast`) over bytes `0..60`. |
+
+---
+
+## 3. Recovery & Truncation Invariants
+
+When initializing `DualMetadataManager`:
+
+1. Both `Slot A` and `Slot B` are decoded and verified against `SLOT_MAGIC`, `SLOT_VERSION`, and their 32-bit IEEE CRC32 checksum.
+2. If both slots are valid, the slot with the higher `epoch` is selected:
+   $$\text{Slot}_{\text{active}} = \arg\max_{S \in \{A, B\}} S.\text{epoch}$$
+3. If one slot was torn during a system crash, its CRC32 check fails and the manager falls back to the intact partner slot.
+4. The companion `.meta.dedup` write-ahead log is truncated to exactly $\text{dedup}_{\text{count}} \times 8$ bytes (`dedup_file.set_len(valid_dedup_bytes)`), discarding any uncommitted partial hash appends before reconstructing `TwoStageDeduplicator::with_committed`.

docs/dataloader/dedup/README.md

@@ -1,66 +1,66 @@
-# Two-Stage XXH3-64 Deduplication & Identity SwissTable
-
-[`data/src/dedup.rs`](https://github.com/cloverx-id/luminamoon/blob/main/data/src/dedup.rs) provides a two-stage streaming deduplicator that separates prefetched records from records committed by training steps.
-
----
-
-## 1. Two-Stage State Machine (`TwoStageDeduplicator`)
-
-In asynchronous pipelines, multiple batches can be staged in memory ahead of the active GPU step. If a single set marked hashes as seen upon prefetch and the process stopped before training consumed them, resuming would either skip unread samples or lose deduplication consistency.
-
-`TwoStageDeduplicator` avoids this by partitioning tracked hashes into two disjoint sets:
-
-$$\mathcal{H}_{\text{committed}} \cap \mathcal{H}_{\text{staged}} = \emptyset$$
-
-```mermaid
-flowchart LR
-    Record["Candidate Record"] --> Hash["Compute XXH3-64 (IdentityHasher)"]
-    Hash --> Drop["Duplicate Digest: Drop Record"]
-    Hash --> Stage["Unique Digest: Insert into staged_lookup"]
-    Stage --> Commit["commit_batch: Promote to committed_hashes & WAL"]
-    Stage --> Evict["rollback_batch: Evict from staged_lookup"]
-```
-
-### Core Operations
-
-1. **`try_stage(batch_id, payload) -> Option<u64>`**:
-   - Computes $h = \text{XXH3}_{64}(\text{payload})$.
-   - Rejects the record (`None`) if $h \in \text{committed}_{\text{hashes}} \cup \text{staged}_{\text{lookup}}$.
-   - Otherwise inserts $h$ into `staged_lookup` and appends $h$ to `staged_batches[batch_id]`.
-2. **`commit_batch(batch_id) -> Vec<u64>`**:
-   - Atomically promotes all hashes in `staged_batches[batch_id]` from `staged_lookup` into `committed_hashes` and returns the vector of promoted hashes to be flushed to the `.meta.dedup` write-ahead log.
-3. **`rollback_batch(batch_id)` & `clear_staged()`**:
-   - Evicts uncommitted batch hashes from `staged_lookup` without touching `committed_hashes`, restoring exact state parity with the last committed GPU step.
-
----
-
-## 2. Zero-Rehash SwissTable (`IdentityBuildHasher`)
-
-Because keys stored in `committed_hashes` and `staged_lookup` are already uniformly distributed 64-bit `xxh3_64` digests, hashing them again with `hashbrown`'s default `foldhash` would waste CPU cycles.
-
-`dedup.rs` defines an `IdentityHasher` that passes the digest directly to the table probe:
-
-```rust
-#[derive(Default)]
-pub struct IdentityHasher(u64);
-
-impl Hasher for IdentityHasher {
-    #[inline]
-    fn finish(&self) -> u64 {
-        self.0
-    }
-
-    #[inline]
-    fn write_u64(&mut self, i: u64) {
-        self.0 = i;
-    }
-}
-
-pub type IdentityBuildHasher = BuildHasherDefault<IdentityHasher>;
-```
-
----
-
-## 3. MinHash LSH Near-Duplicate Shingling (`src/minhash.rs`)
-
-For fuzzy near-duplicate rejection alongside exact 64-bit `xxh3_64` deduplication, `MinHashDeduplicator` computes $K$-character UTF-8 shingles hashed across $P$ deterministic permuted `xxh3_64_with_seed` lanes and estimates pairwise Jaccard similarity against indexed document signatures.
+# Two-Stage XXH3-64 Deduplication & Identity SwissTable
+
+[`data/src/dedup.rs`](https://github.com/cloverx-id/luminamoon/blob/main/data/src/dedup.rs) provides a two-stage streaming deduplicator that separates prefetched records from records committed by training steps.
+
+---
+
+## 1. Two-Stage State Machine (`TwoStageDeduplicator`)
+
+In asynchronous pipelines, multiple batches can be staged in memory ahead of the active GPU step. If a single set marked hashes as seen upon prefetch and the process stopped before training consumed them, resuming would either skip unread samples or lose deduplication consistency.
+
+`TwoStageDeduplicator` avoids this by partitioning tracked hashes into two disjoint sets:
+
+$$\mathcal{H}_{\text{committed}} \cap \mathcal{H}_{\text{staged}} = \emptyset$$
+
+```mermaid
+flowchart LR
+    Record["Candidate Record"] --> Hash["Compute XXH3-64 (IdentityHasher)"]
+    Hash --> Drop["Duplicate Digest: Drop Record"]
+    Hash --> Stage["Unique Digest: Insert into staged_lookup"]
+    Stage --> Commit["commit_batch: Promote to committed_hashes & WAL"]
+    Stage --> Evict["rollback_batch: Evict from staged_lookup"]
+```
+
+### Core Operations
+
+1. **`try_stage(batch_id, payload) -> Option<u64>`**:
+   - Computes $h = \text{XXH3}_{64}(\text{payload})$.
+   - Rejects the record (`None`) if $h \in \text{committed}_{\text{hashes}} \cup \text{staged}_{\text{lookup}}$.
+   - Otherwise inserts $h$ into `staged_lookup` and appends $h$ to `staged_batches[batch_id]`.
+2. **`commit_batch(batch_id) -> Vec<u64>`**:
+   - Atomically promotes all hashes in `staged_batches[batch_id]` from `staged_lookup` into `committed_hashes` and returns the vector of promoted hashes to be flushed to the `.meta.dedup` write-ahead log.
+3. **`rollback_batch(batch_id)` & `clear_staged()`**:
+   - Evicts uncommitted batch hashes from `staged_lookup` without touching `committed_hashes`, restoring exact state parity with the last committed GPU step.
+
+---
+
+## 2. Zero-Rehash SwissTable (`IdentityBuildHasher`)
+
+Because keys stored in `committed_hashes` and `staged_lookup` are already uniformly distributed 64-bit `xxh3_64` digests, hashing them again with `hashbrown`'s default `foldhash` would waste CPU cycles.
+
+`dedup.rs` defines an `IdentityHasher` that passes the digest directly to the table probe:
+
+```rust
+#[derive(Default)]
+pub struct IdentityHasher(u64);
+
+impl Hasher for IdentityHasher {
+    #[inline]
+    fn finish(&self) -> u64 {
+        self.0
+    }
+
+    #[inline]
+    fn write_u64(&mut self, i: u64) {
+        self.0 = i;
+    }
+}
+
+pub type IdentityBuildHasher = BuildHasherDefault<IdentityHasher>;
+```
+
+---
+
+## 3. MinHash LSH Near-Duplicate Shingling (`src/minhash.rs`)
+
+For fuzzy near-duplicate rejection alongside exact 64-bit `xxh3_64` deduplication, `MinHashDeduplicator` computes $K$-character UTF-8 shingles hashed across $P$ deterministic permuted `xxh3_64_with_seed` lanes and estimates pairwise Jaccard similarity against indexed document signatures.

docs/dataloader/multimodal_and_hf_hub/README.md

@@ -1,38 +1,38 @@
-# Columnar Serialization (Parquet, Arrow IPC & JSONL)
-
-[`data/src/serialize.rs`](https://github.com/cloverx-id/luminamoon/blob/main/data/src/serialize.rs) and [`python/luminamoon/data/serialization.py`](https://github.com/cloverx-id/luminamoon/blob/main/python/luminamoon/data/serialization.py) serialize Python dictionaries and columnar tables into Apache Parquet (ZSTD), Apache Arrow IPC (`ARROW1`), and line-delimited JSONL (`NDJSON`) with the Python GIL released.
-
----
-
-## 1. Unified Columnar Type System (`DataValue`)
-
-`data/src/serialize.rs` maps Python sequences and dictionaries into an 8-variant `DataValue` enum:
-
-| Variant | Python Input Types | Arrow / Parquet Target Type (`hf_image_struct = true`) |
-| :--- | :--- | :--- |
-| `DataValue::Null` | `None` | Nullable slot in target column array |
-| `DataValue::Text(String)` | `str` (non-JSON) | `DataType::Utf8` (`StringArray`) |
-| `DataValue::Bytes(Vec<u8>)` | `bytes`, `bytearray` | `DataType::Struct(bytes: Binary, path: Utf8)` (or `DataType::Binary` when `hf_image_struct = false`) |
-| `DataValue::Json(String)` | Valid JSON object/array `str` | `DataType::Utf8` in Parquet/Arrow; parsed JSON value in `.jsonl` |
-| `DataValue::Int(i64)` | `int` | `DataType::Int64` (`Int64Array`) |
-| `DataValue::Float(f64)` | `float` | `DataType::Float64` (`Float64Array`) |
-| `DataValue::Bool(bool)` | `bool` | `DataType::Boolean` (`BooleanArray`) |
-| `DataValue::List(Vec<DataValue>)` | Nested sequences | JSON-encoded array representation |
-
----
-
-## 2. HuggingFace Dataset Viewer Image Struct Compatibility
-
-When `hf_image_struct = True` (default in `serialize_to_parquet`), binary columns (`DataValue::Bytes`) are automatically encoded as an Arrow `StructArray` with schema:
-
-$$\text{Struct}\langle\text{bytes}: \text{Binary},\ \text{path}: \text{Utf8}\rangle$$
-
-This matches the standard HuggingFace Dataset Viewer image/audio schema, enabling inline previews on `huggingface.co` without post-processing.
-
----
-
-## 3. Public Python Serialization Helpers (`serialization.py`)
-
-- **`serialize_to_parquet(columns, output_path=None, hf_image_struct=True) -> bytes`**: Encodes columnar data into ZSTD-compressed Apache Parquet bytes (`PAR1`) with the GIL released (`py.detach()`).
-- **`serialize_to_arrow(columns, output_path=None) -> bytes`**: Encodes columnar data into Apache Arrow IPC File Format bytes (`ARROW1`).
-- **`serialize_to_jsonl(columns, output_path=None) -> bytes`**: Encodes columnar data into UTF-8 line-delimited JSONL records.
+# Columnar Serialization (Parquet, Arrow IPC & JSONL)
+
+[`data/src/serialize.rs`](https://github.com/cloverx-id/luminamoon/blob/main/data/src/serialize.rs) and [`python/luminamoon/data/serialization.py`](https://github.com/cloverx-id/luminamoon/blob/main/python/luminamoon/data/serialization.py) serialize Python dictionaries and columnar tables into Apache Parquet (ZSTD), Apache Arrow IPC (`ARROW1`), and line-delimited JSONL (`NDJSON`) with the Python GIL released.
+
+---
+
+## 1. Unified Columnar Type System (`DataValue`)
+
+`data/src/serialize.rs` maps Python sequences and dictionaries into an 8-variant `DataValue` enum:
+
+| Variant | Python Input Types | Arrow / Parquet Target Type (`hf_image_struct = true`) |
+| :--- | :--- | :--- |
+| `DataValue::Null` | `None` | Nullable slot in target column array |
+| `DataValue::Text(String)` | `str` (non-JSON) | `DataType::Utf8` (`StringArray`) |
+| `DataValue::Bytes(Vec<u8>)` | `bytes`, `bytearray` | `DataType::Struct(bytes: Binary, path: Utf8)` (or `DataType::Binary` when `hf_image_struct = false`) |
+| `DataValue::Json(String)` | Valid JSON object/array `str` | `DataType::Utf8` in Parquet/Arrow; parsed JSON value in `.jsonl` |
+| `DataValue::Int(i64)` | `int` | `DataType::Int64` (`Int64Array`) |
+| `DataValue::Float(f64)` | `float` | `DataType::Float64` (`Float64Array`) |
+| `DataValue::Bool(bool)` | `bool` | `DataType::Boolean` (`BooleanArray`) |
+| `DataValue::List(Vec<DataValue>)` | Nested sequences | JSON-encoded array representation |
+
+---
+
+## 2. HuggingFace Dataset Viewer Image Struct Compatibility
+
+When `hf_image_struct = True` (default in `serialize_to_parquet`), binary columns (`DataValue::Bytes`) are automatically encoded as an Arrow `StructArray` with schema:
+
+$$\text{Struct}\langle\text{bytes}: \text{Binary},\ \text{path}: \text{Utf8}\rangle$$
+
+This matches the standard HuggingFace Dataset Viewer image/audio schema, enabling inline previews on `huggingface.co` without post-processing.
+
+---
+
+## 3. Public Python Serialization Helpers (`serialization.py`)
+
+- **`serialize_to_parquet(columns, output_path=None, hf_image_struct=True) -> bytes`**: Encodes columnar data into ZSTD-compressed Apache Parquet bytes (`PAR1`) with the GIL released (`py.detach()`).
+- **`serialize_to_arrow(columns, output_path=None) -> bytes`**: Encodes columnar data into Apache Arrow IPC File Format bytes (`ARROW1`).
+- **`serialize_to_jsonl(columns, output_path=None) -> bytes`**: Encodes columnar data into UTF-8 line-delimited JSONL records.

docs/dataloader/preram/README.md

@@ -1,37 +1,37 @@
-# Dual-Layout StringTensor & Async Resumable Pipelines
-
-[`data/src/string_tensor.rs`](https://github.com/cloverx-id/luminamoon/blob/main/data/src/string_tensor.rs) and [`python/luminamoon/data/adapters.py`](https://github.com/cloverx-id/luminamoon/blob/main/python/luminamoon/data/adapters.py) provide complementary CPU and GPU string layouts alongside asynchronous background batch prefetching for PyTorch and standalone training loops.
-
----
-
-## 1. Dual-Layout `StringTensor` (`src/string_tensor.rs`)
-
-`StringTensor` bridges CPU string processing and GPU tensor ingestion via two complementary memory representations:
-
-### 1.1 CPU Layout: 16-Byte Umbra / DuckDB `StringView16`
-```rust
-#[repr(C)]
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
-pub struct StringView16 {
-    pub(crate) len: u32,
-    pub(crate) prefix_or_inline: [u8; 4],
-    pub(crate) suffix_or_offset: [u8; 8],
-}
-```
-
-- **Inlined Strings ($\text{len} \le 12$ bytes)**: Stored completely inside the 16-byte struct (`prefix_or_inline` + `suffix_or_offset`) with zero heap pointer dereferences.
-- **Pool-Referenced Strings ($\text{len} > 12$ bytes)**: Stores the first 4 bytes in `prefix_or_inline` (enabling single-register 32-bit prefix rejection in `filter_by_prefix` and `fast_eq`), alongside a 4-byte `offset` and 4-byte `buffer_id` into `Arc<Vec<u8>>`.
-
-### 1.2 GPU & Arrow Layout: Contiguous Offsets + Payload
-- `to_gpu_buffers()` emits a contiguous `uint32` offset array `[0, off_1, ..., off_N]` of length $N + 1$ and a flat `uint8` UTF-8 byte buffer.
-- Directly exported to PyTorch (`to_torch_gpu(device)`), NumPy (`to_numpy_gpu()`), Apache Arrow (`to_arrow(zero_copy=True)` via `pa.StringArray.from_buffers`), and Polars (`to_polars()`).
-
----
-
-## 2. Asynchronous Prefetching & PyTorch Integration (`adapters.py`)
-
-1. **`ResumablePipeline`**:
-   - Wraps any Python iterable or streaming dataset with single-pass `FusedPipeline` execution, `StateTracker` 128-byte Ping-Pong coordinate tracking, and background thread prefetching (`prefetch_batches > 0` via bounded `queue.Queue`).
-   - Calling `pipe.commit()` atomically persists both the active `StateTracker` batch coordinate (`Slot A` / `Slot B`) and `StreamDeduplicator` WAL.
-2. **`create_resumable_collate`**:
-   - Generates a drop-in `collate_fn` for `torch.utils.data.DataLoader` that extracts text fields, runs `FusedPipeline` (or modular `TextCleaner` -> `QualityFilter` -> `StreamDeduplicator` -> `PIIMasker`), and returns either tokenized PyTorch tensors or a `StringTensor`.
+# Dual-Layout StringTensor & Async Resumable Pipelines
+
+[`data/src/string_tensor.rs`](https://github.com/cloverx-id/luminamoon/blob/main/data/src/string_tensor.rs) and [`python/luminamoon/data/adapters.py`](https://github.com/cloverx-id/luminamoon/blob/main/python/luminamoon/data/adapters.py) provide complementary CPU and GPU string layouts alongside asynchronous background batch prefetching for PyTorch and standalone training loops.
+
+---
+
+## 1. Dual-Layout `StringTensor` (`src/string_tensor.rs`)
+
+`StringTensor` bridges CPU string processing and GPU tensor ingestion via two complementary memory representations:
+
+### 1.1 CPU Layout: 16-Byte Umbra / DuckDB `StringView16`
+```rust
+#[repr(C)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub struct StringView16 {
+    pub(crate) len: u32,
+    pub(crate) prefix_or_inline: [u8; 4],
+    pub(crate) suffix_or_offset: [u8; 8],
+}
+```
+
+- **Inlined Strings ($\text{len} \le 12$ bytes)**: Stored completely inside the 16-byte struct (`prefix_or_inline` + `suffix_or_offset`) with zero heap pointer dereferences.
+- **Pool-Referenced Strings ($\text{len} > 12$ bytes)**: Stores the first 4 bytes in `prefix_or_inline` (enabling single-register 32-bit prefix rejection in `filter_by_prefix` and `fast_eq`), alongside a 4-byte `offset` and 4-byte `buffer_id` into `Arc<Vec<u8>>`.
+
+### 1.2 GPU & Arrow Layout: Contiguous Offsets + Payload
+- `to_gpu_buffers()` emits a contiguous `uint32` offset array `[0, off_1, ..., off_N]` of length $N + 1$ and a flat `uint8` UTF-8 byte buffer.
+- Directly exported to PyTorch (`to_torch_gpu(device)`), NumPy (`to_numpy_gpu()`), Apache Arrow (`to_arrow(zero_copy=True)` via `pa.StringArray.from_buffers`), and Polars (`to_polars()`).
+
+---
+
+## 2. Asynchronous Prefetching & PyTorch Integration (`adapters.py`)
+
+1. **`ResumablePipeline`**:
+   - Wraps any Python iterable or streaming dataset with single-pass `FusedPipeline` execution, `StateTracker` 128-byte Ping-Pong coordinate tracking, and background thread prefetching (`prefetch_batches > 0` via bounded `queue.Queue`).
+   - Calling `pipe.commit()` atomically persists both the active `StateTracker` batch coordinate (`Slot A` / `Slot B`) and `StreamDeduplicator` WAL.
+2. **`create_resumable_collate`**:
+   - Generates a drop-in `collate_fn` for `torch.utils.data.DataLoader` that extracts text fields, runs `FusedPipeline` (or modular `TextCleaner` -> `QualityFilter` -> `StreamDeduplicator` -> `PIIMasker`), and returns either tokenized PyTorch tensors or a `StringTensor`.

docs/dataloader/python_ffi_and_runtime/README.md

@@ -1,29 +1,29 @@
-# PyO3 FFI Bridge & Modular Python Runtime
-
-[`data/src/py_bridge.rs`](https://github.com/cloverx-id/luminamoon/blob/main/data/src/py_bridge.rs) and the modular [`python/luminamoon/data/`](https://github.com/cloverx-id/luminamoon/blob/main/python/luminamoon/data/__init__.py) package connect the native Rust SIMD and columnar engine with Python under the CPython Stable ABI (`abi3-py310` and free-threaded `abi3t`).
-
----
-
-## 1. Native PyO3 Classes & Functions (`src/py_bridge.rs`)
-
-All native symbols are compiled into `_luminamoon_optimizer_rs` and re-exported through `python/luminamoon/data/_ffi.py`:
-
-| Rust PyO3 Symbol | Python Wrapper (`luminamoon.data`) | GIL Detached (`py.detach`) | Core Responsibility |
-| :--- | :--- | :--- | :--- |
-| `PyStreamCoordinate` | `StreamCoordinate` (`checkpoint.py`) | N/A (32B value) | 32-byte little-endian stream coordinate with `to_bytes()` / `from_bytes()`. |
-| `PyStateTracker` | `StateTracker` (`checkpoint.py`) | Yes | 128-byte Dual-Slot Ping-Pong (`Slot A` / `Slot B`) atomic checkpoint manager (`read`, `update`, `commit`). |
-| `PyStreamDeduplicator` | `StreamDeduplicator` (`dedup.py`) | Yes | Two-stage `xxh3_64` SwissTable deduplicator with `.dedup` WAL (`is_unique`, `filter_batch`, `commit`, `rollback`). |
-| `PyMinHashDeduplicator` | `MinHashDeduplicator` (`minhash.py`) | Yes | 64-permutation MinHash LSH near-duplicate detector (`is_unique`, `similarity`, `filter_batch`). |
-| `PyQualityFilter` | `QualityFilter` (`quality.py`) | Yes | Parallel Rayon Shannon entropy, repetition, and symbol ratio filter (`is_high_quality`, `metrics`, `filter_batch`). |
-| `PyStringTensor` | `StringTensor` (`string_tensor.py`) | Yes | Dual-layout 16B CPU `StringView16` and contiguous GPU/Arrow buffer container. |
-| `fused_pipeline_process_batch` | `FusedPipeline` (`fused.py`) | Yes | Single-pass Rayon + SIMD text cleaning, quality filtering, and deduplication returning `StringTensor`. |
-| `serialize_table_to_bytes` | `serialize_to_*` (`serialization.py`) | Yes | Columnar serialization to Parquet (`PAR1`), Arrow IPC (`ARROW1`), and JSONL. |
-
----
-
-## 2. Modular Pure-Python Fallback Layer (`python/luminamoon/data/fallbacks/`)
-
-When running in environments without the compiled native binary, `_ffi.py` transparently falls back to the modular `fallbacks/` subpackage:
-
-- `cleaner.py`, `quality.py`, `dedup.py`, `minhash.py`, `masker.py`, `coordinate.py`, `tracker.py`, `string_tensor.py`, `fused.py`, `serialization.py`, and `helpers.py`.
-- Preserves API and binary format compatibility (including 32-byte `StreamCoordinate` and 128-byte IEEE CRC32 Ping-Pong `.meta` checkpoints).
+# PyO3 FFI Bridge & Modular Python Runtime
+
+[`data/src/py_bridge.rs`](https://github.com/cloverx-id/luminamoon/blob/main/data/src/py_bridge.rs) and the modular [`python/luminamoon/data/`](https://github.com/cloverx-id/luminamoon/blob/main/python/luminamoon/data/__init__.py) package connect the native Rust SIMD and columnar engine with Python under the CPython Stable ABI (`abi3-py310` and free-threaded `abi3t`).
+
+---
+
+## 1. Native PyO3 Classes & Functions (`src/py_bridge.rs`)
+
+All native symbols are compiled into `_luminamoon_optimizer_rs` and re-exported through `python/luminamoon/data/_ffi.py`:
+
+| Rust PyO3 Symbol | Python Wrapper (`luminamoon.data`) | GIL Detached (`py.detach`) | Core Responsibility |
+| :--- | :--- | :--- | :--- |
+| `PyStreamCoordinate` | `StreamCoordinate` (`checkpoint.py`) | N/A (32B value) | 32-byte little-endian stream coordinate with `to_bytes()` / `from_bytes()`. |
+| `PyStateTracker` | `StateTracker` (`checkpoint.py`) | Yes | 128-byte Dual-Slot Ping-Pong (`Slot A` / `Slot B`) atomic checkpoint manager (`read`, `update`, `commit`). |
+| `PyStreamDeduplicator` | `StreamDeduplicator` (`dedup.py`) | Yes | Two-stage `xxh3_64` SwissTable deduplicator with `.dedup` WAL (`is_unique`, `filter_batch`, `commit`, `rollback`). |
+| `PyMinHashDeduplicator` | `MinHashDeduplicator` (`minhash.py`) | Yes | 64-permutation MinHash LSH near-duplicate detector (`is_unique`, `similarity`, `filter_batch`). |
+| `PyQualityFilter` | `QualityFilter` (`quality.py`) | Yes | Parallel Rayon Shannon entropy, repetition, and symbol ratio filter (`is_high_quality`, `metrics`, `filter_batch`). |
+| `PyStringTensor` | `StringTensor` (`string_tensor.py`) | Yes | Dual-layout 16B CPU `StringView16` and contiguous GPU/Arrow buffer container. |
+| `fused_pipeline_process_batch` | `FusedPipeline` (`fused.py`) | Yes | Single-pass Rayon + SIMD text cleaning, quality filtering, and deduplication returning `StringTensor`. |
+| `serialize_table_to_bytes` | `serialize_to_*` (`serialization.py`) | Yes | Columnar serialization to Parquet (`PAR1`), Arrow IPC (`ARROW1`), and JSONL. |
+
+---
+
+## 2. Modular Pure-Python Fallback Layer (`python/luminamoon/data/fallbacks/`)
+
+When running in environments without the compiled native binary, `_ffi.py` transparently falls back to the modular `fallbacks/` subpackage:
+
+- `cleaner.py`, `quality.py`, `dedup.py`, `minhash.py`, `masker.py`, `coordinate.py`, `tracker.py`, `string_tensor.py`, `fused.py`, `serialization.py`, and `helpers.py`.
+- Preserves API and binary format compatibility (including 32-byte `StreamCoordinate` and 128-byte IEEE CRC32 Ping-Pong `.meta` checkpoints).

docs/dataloader/simd_and_postprocess/README.md

@@ -1,62 +1,62 @@
-# CPU SIMD Kernels & Rayon Normalization
-
-[`data/src/simd_cpu.rs`](https://github.com/cloverx-id/luminamoon/blob/main/data/src/simd_cpu.rs) and [`data/src/postprocess.rs`](https://github.com/cloverx-id/luminamoon/blob/main/data/src/postprocess.rs) provide vectorized byte scanning and multi-core text normalization without external crate dependencies.
-
----
-
-## 1. Instruction Set Support & Vector Widths (`src/simd_cpu.rs`)
-
-`luminamoon-dataloader` detects host CPU capabilities at startup and caches the active `CpuIsa` in a lock-free `OnceLock<CpuIsa>`:
-
-| Architecture Target | Enum Variant | Registers | Bytes / Cycle | Intrinsics Used |
-| :--- | :--- | :--- | :--- | :--- |
-| **x86 / x86_64 AVX-512BW** | `CpuIsa::Avx512` | ZMM (`__m512i`) | 64 bytes | `_mm512_set1_epi8`, `_mm512_loadu_si512`, `_mm512_cmpeq_epi8_mask` |
-| **x86 / x86_64 AVX2** | `CpuIsa::Avx2` | YMM (`__m256i`) | 32 bytes | `_mm256_set1_epi8`, `_mm256_loadu_si256`, `_mm256_cmpeq_epi8`, `_mm256_movemask_epi8` |
-| **ARM64 NEON** | `CpuIsa::Neon` | V-Reg (`uint8x16_t`) | 16 bytes | `vdupq_n_u8`, `vld1q_u8`, `vceqq_u8`, `vmaxvq_u8` |
-| **Portable Fallback** | `CpuIsa::Scalar` | General Purpose | 1 byte | Iterator `position` / `rposition` |
-
----
-
-## 2. Vectorized Byte Scanning Kernels
-
-1. **`find_byte_fast(haystack: &[u8], needle: u8) -> Option<usize>`**:
-   - Scans forward in 64-byte (`AVX-512BW`), 32-byte (`AVX2`), or 16-byte (`NEON`) unaligned chunks, locating the first matching byte offset via hardware bitmask `trailing_zeros()`.
-   - Powers `find_newline_fast` for `.jsonl`/`.txt` line splitting and whitespace boundary detection.
-2. **`rfind_byte_fast(haystack: &[u8], needle: u8) -> Option<usize>`**:
-   - Scans backward from the end of a byte slice using `leading_zeros()` on the comparison mask.
-   - Powers `rfind_newline_fast` to locate the exact last complete line boundary in a byte buffer in $O(1)$ vector steps.
-3. **`contains_byte_fast(haystack: &[u8], needle: u8) -> bool`**:
-   - Fast-path guard in `postprocess.rs` that skips string reallocation during CRLF (`\r\n`) normalization whenever a record contains no `\r` (`0x0D`) byte.
-
----
-
-## 3. 64-Byte Cache-Line Aligned Buffer (`CacheAlignedBuffer`)
-
-```rust
-#[repr(align(64))]
-#[derive(Debug, Clone)]
-pub struct CacheAlignedBuffer {
-    pub data: Vec<u8>,
-}
-```
-
-Rounds buffer capacities up to 64-byte cache-line multiples (`(cap + 63) & !63`) to avoid split-line penalties during SIMD scanning and batch serialization.
-
----
-
-## 4. Parallel Rayon Normalization Pipeline (`src/postprocess.rs`)
-
-`TextCleaner` and `FusedPipeline` govern parallel per-record sanitization across raw text batches:
-
-```mermaid
-flowchart LR
-    Input["Raw Shard Bytes"] --> Scan["SIMD Newline Scan (AVX-512BW / AVX2 / NEON)"]
-    Scan --> Extract["Rayon Parallel JSON Field Extraction"]
-    Extract --> Clean["Fast CRLF Normalization & Whitespace Trim"]
-    Clean --> Output["Valid Length: Normalized UTF-8 Batch"]
-    Clean --> Truncate["Exceeds Max: Snap to UTF-8 Boundary"]
-    Truncate --> Output
-    Clean --> Drop["Below Min / Empty: Filter Out Record"]
-```
-
-- **Order-Preserving Parallel Execution**: `process_batch_parallel_indexed` uses Rayon's indexed `par_iter().map(...)` to normalize batches across all CPU cores while preserving exact deterministic record ordering for `StreamCoordinate` progression.
+# CPU SIMD Kernels & Rayon Normalization
+
+[`data/src/simd_cpu.rs`](https://github.com/cloverx-id/luminamoon/blob/main/data/src/simd_cpu.rs) and [`data/src/postprocess.rs`](https://github.com/cloverx-id/luminamoon/blob/main/data/src/postprocess.rs) provide vectorized byte scanning and multi-core text normalization without external crate dependencies.
+
+---
+
+## 1. Instruction Set Support & Vector Widths (`src/simd_cpu.rs`)
+
+`luminamoon-dataloader` detects host CPU capabilities at startup and caches the active `CpuIsa` in a lock-free `OnceLock<CpuIsa>`:
+
+| Architecture Target | Enum Variant | Registers | Bytes / Cycle | Intrinsics Used |
+| :--- | :--- | :--- | :--- | :--- |
+| **x86 / x86_64 AVX-512BW** | `CpuIsa::Avx512` | ZMM (`__m512i`) | 64 bytes | `_mm512_set1_epi8`, `_mm512_loadu_si512`, `_mm512_cmpeq_epi8_mask` |
+| **x86 / x86_64 AVX2** | `CpuIsa::Avx2` | YMM (`__m256i`) | 32 bytes | `_mm256_set1_epi8`, `_mm256_loadu_si256`, `_mm256_cmpeq_epi8`, `_mm256_movemask_epi8` |
+| **ARM64 NEON** | `CpuIsa::Neon` | V-Reg (`uint8x16_t`) | 16 bytes | `vdupq_n_u8`, `vld1q_u8`, `vceqq_u8`, `vmaxvq_u8` |
+| **Portable Fallback** | `CpuIsa::Scalar` | General Purpose | 1 byte | Iterator `position` / `rposition` |
+
+---
+
+## 2. Vectorized Byte Scanning Kernels
+
+1. **`find_byte_fast(haystack: &[u8], needle: u8) -> Option<usize>`**:
+   - Scans forward in 64-byte (`AVX-512BW`), 32-byte (`AVX2`), or 16-byte (`NEON`) unaligned chunks, locating the first matching byte offset via hardware bitmask `trailing_zeros()`.
+   - Powers `find_newline_fast` for `.jsonl`/`.txt` line splitting and whitespace boundary detection.
+2. **`rfind_byte_fast(haystack: &[u8], needle: u8) -> Option<usize>`**:
+   - Scans backward from the end of a byte slice using `leading_zeros()` on the comparison mask.
+   - Powers `rfind_newline_fast` to locate the exact last complete line boundary in a byte buffer in $O(1)$ vector steps.
+3. **`contains_byte_fast(haystack: &[u8], needle: u8) -> bool`**:
+   - Fast-path guard in `postprocess.rs` that skips string reallocation during CRLF (`\r\n`) normalization whenever a record contains no `\r` (`0x0D`) byte.
+
+---
+
+## 3. 64-Byte Cache-Line Aligned Buffer (`CacheAlignedBuffer`)
+
+```rust
+#[repr(align(64))]
+#[derive(Debug, Clone)]
+pub struct CacheAlignedBuffer {
+    pub data: Vec<u8>,
+}
+```
+
+Rounds buffer capacities up to 64-byte cache-line multiples (`(cap + 63) & !63`) to avoid split-line penalties during SIMD scanning and batch serialization.
+
+---
+
+## 4. Parallel Rayon Normalization Pipeline (`src/postprocess.rs`)
+
+`TextCleaner` and `FusedPipeline` govern parallel per-record sanitization across raw text batches:
+
+```mermaid
+flowchart LR
+    Input["Raw Shard Bytes"] --> Scan["SIMD Newline Scan (AVX-512BW / AVX2 / NEON)"]
+    Scan --> Extract["Rayon Parallel JSON Field Extraction"]
+    Extract --> Clean["Fast CRLF Normalization & Whitespace Trim"]
+    Clean --> Output["Valid Length: Normalized UTF-8 Batch"]
+    Clean --> Truncate["Exceeds Max: Snap to UTF-8 Boundary"]
+    Truncate --> Output
+    Clean --> Drop["Below Min / Empty: Filter Out Record"]
+```
+
+- **Order-Preserving Parallel Execution**: `process_batch_parallel_indexed` uses Rayon's indexed `par_iter().map(...)` to normalize batches across all CPU cores while preserving exact deterministic record ordering for `StreamCoordinate` progression.