Add benchmarking script

BENCHMARKING.md

@@ -0,0 +1,26 @@
+# Benchmarking models
+
+To use `bench-TriLMs.sh`, you need to
+
+- Place it in a `llama.cpp` checkout
+- Have `cmake`, `gcc`, and other dependencies of `llama.cpp`
+- If you want to benchmark on GPUs, the script checks if `nvidia-smi` is present, and you'll also need the necessary compile-time dependencies
+
+The script will automatically download the models and quantize different variants.
+
+It will then produce 2 result files, one called `results-$(date +%s).json` and the other called `results-$(date +%s)-cpuinfo.txt`. Both will use the exact same date.
+
+The intention is to eventually read the produced `.json` in a Python script with
+
+```python3
+from __future__ import annotations
+
+from typing import Any
+import json
+
+with open("result-1234567890.json") as f:
+    data: list[list[dict[str, Any]]] = json.loads("[" + f.read() + "]")
+
+    # Then use that data
+    ...
+```

bench-TriLMs.sh

@@ -0,0 +1,101 @@
+#!/usr/bin/env bash
+set -eux
+
+cd "$(dirname "$0")"
+
+MODEL_DIR="bench-TriLMs-models"
+LLAMA_CPP_PATH="."
+sizes=("1.5" "2.4" "3.9")
+types=("TQ1_0" "TQ2_0" "Q8_0" "F16" "BF16")
+gputypes=("Q8_0" "F16" "BF16")
+
+function gather_models() {
+  echo Gather the models
+  if [ ! -d "$MODEL_DIR" ]; then
+    mkdir -p -- "$MODEL_DIR"
+  fi
+  (
+    cd "$MODEL_DIR"
+    for sz in "${sizes[@]}"; do
+      filename="TriLM_${sz}B_Unpacked-TQ1_0-F16.gguf"
+      if [ ! -f "$filename" ]; then
+        wget "https://huggingface.co/compilade/quant-tests/resolve/main/${filename}"
+      fi
+    done
+  )
+}
+
+function build_llama_cpp() {
+  echo Build llama.cpp for CPU
+
+  (
+    cd -- "$LLAMA_CPP_PATH"
+    if [ -d build ]; then
+      pwd
+      echo 'rm -rI build'
+      rm -rI build
+    fi
+    mkdir build
+    cd build
+    cmake .. "$@"
+    make -j llama-bench llama-quantize
+  )
+}
+
+function quantize() {
+  echo "Make all model types we'll test"
+  (
+    for sz in "${sizes[@]}"; do
+      for ty in "${types[@]}"; do
+        filenames=("$MODEL_DIR"/TriLM_"${sz}"B_Unpacked-{TQ1_0-F16,"$ty"}.gguf)
+        if [ ! -f "${filenames[1]}" ]; then
+          "$LLAMA_CPP_PATH"/build/bin/llama-quantize --allow-requantize "${filenames[@]}" "$ty"
+        fi
+      done
+    done
+  )
+}
+
+function bench() {
+  echo Test each model one by one for different numbers of threads
+
+  for sz in "${sizes[@]}"; do
+    for ty in "$@"; do
+      for th in 1 2 4 8; do
+        {
+          "$LLAMA_CPP_PATH"/build/bin/llama-bench -v -m "${MODEL_DIR}/TriLM_${sz}B_Unpacked-${ty}.gguf" -t "${th}" -p 512 -n 128 -r 4 -o json
+          printf "%s\n" ","
+        }
+      done
+    done
+  done
+}
+
+function bench_cpu() {
+  bench "${types[@]}" >> "$1"
+}
+
+function bench_gpu() {
+  bench "${gputypes[@]}" >> "$1"
+}
+
+currentTime="$(date +'%s')"
+resultFile="results-${currentTime}.json"
+infoFile="results-${currentTime}-info.txt"
+lscpu > "$infoFile"
+
+gather_models
+build_llama_cpp -DGGML_NATIVE=ON -DGGML_CPU=ON
+quantize
+
+echo "---" >> "$infoFile"
+ls -go "$MODEL_DIR" >> "$infoFile"
+
+bench_cpu "$resultFile"
+
+if [ -x "$(command -v nvidia-smi)" ]; then
+  echo GPU detected, benchark with that too.
+  build_llama_cpp -DGGML_NATIVE=ON -DGGML_CUDA=ON -DGGML_CUDA_F16=ON
+  bench_gpu "$resultFile"
+fi
+