## TESTING https://github.com/ikawrakow/ik_llama.cpp/discussions/477#discussioncomment-13361171 # --threads 18 CUDA_DEVICE_ORDER=PCI_BUS_ID \ CUDA_VISIBLE_DEVICES=0 \ ~/ik_llama-main-b3746-f26fe36-bin-win-cuda-12.8-x64/llama-sweep-bench \ -m DeepSeek-R1-0528-IQ1_S_R4-00001-of-00003.gguf \ -c 8192 \ -mla 3 -fa \ -amb 512 \ -fmoe \ -ngl 99 \ -ot exps=CPU \ --warmup-batch \ --threads 18 --- main: n_kv_max = 8192, n_batch = 2048, n_ubatch = 512, flash_attn = 1, n_gpu_layers = 99, n_threads = 18, n_threads_batch = 18 | PP | TG | N_KV | T_PP s | S_PP t/s | T_TG s | S_TG t/s | |-------|--------|--------|----------|----------|----------|----------| | 512 | 128 | 0 | 33.966 | 15.07 | 39.843 | 3.21 | | 512 | 128 | 512 | 33.340 | 15.36 | 39.279 | 3.26 | --- # --threads 18 + -ub 1024 -b 1024 CUDA_DEVICE_ORDER=PCI_BUS_ID \ CUDA_VISIBLE_DEVICES=0 \ ~/ik_llama-main-b3746-f26fe36-bin-win-cuda-12.8-x64/llama-sweep-bench \ -m DeepSeek-R1-0528-IQ1_S_R4-00001-of-00003.gguf \ -c 8192 \ -mla 3 -fa \ -amb 512 \ -fmoe \ -ngl 99 \ -ot exps=CPU \ --warmup-batch \ --threads 18 \ -ub 1024 -b 1024 --- main: n_kv_max = 8192, n_batch = 1024, n_ubatch = 1024, flash_attn = 1, n_gpu_layers = 99, n_threads = 18, n_threads_batch = 18 | PP | TG | N_KV | T_PP s | S_PP t/s | T_TG s | S_TG t/s | |-------|--------|--------|----------|----------|----------|----------| | 1024 | 256 | 0 | 59.474 | 17.22 | 79.560 | 3.22 | | 1024 | 256 | 1024 | 61.131 | 16.75 | 79.187 | 3.23 | --- # --threads 18 + -ub 2028 -b 2048 CUDA_DEVICE_ORDER=PCI_BUS_ID \ CUDA_VISIBLE_DEVICES=0 \ ~/ik_llama-main-b3746-f26fe36-bin-win-cuda-12.8-x64/llama-sweep-bench \ -m DeepSeek-R1-0528-IQ1_S_R4-00001-of-00003.gguf \ -c 8192 \ -mla 3 -fa \ -amb 512 \ -fmoe \ -ngl 99 \ -ot exps=CPU \ --warmup-batch \ --threads 18 \ -ub 2028 -b 2048 --- main: n_kv_max = 8192, n_batch = 2048, n_ubatch = 2028, flash_attn = 1, n_gpu_layers = 99, n_threads = 18, n_threads_batch = 18 | PP | TG | N_KV | T_PP s | S_PP t/s | T_TG s | S_TG t/s | |-------|--------|--------|----------|----------|----------|----------| | 2028 | 507 | 0 | 109.917 | 18.45 | 156.104 | 3.25 | | 2028 | 507 | 2028 | 109.416 | 18.53 | 156.624 | 3.24 | --- # --threads 36 CUDA_DEVICE_ORDER=PCI_BUS_ID \ CUDA_VISIBLE_DEVICES=0 \ ~/ik_llama-main-b3746-f26fe36-bin-win-cuda-12.8-x64/llama-sweep-bench \ -m DeepSeek-R1-0528-IQ1_S_R4-00001-of-00003.gguf \ -c 8192 \ -mla 3 -fa \ -amb 512 \ -fmoe \ -ngl 99 \ -ot exps=CPU \ --warmup-batch \ --threads 36 --- main: n_kv_max = 8192, n_batch = 2048, n_ubatch = 512, flash_attn = 1, n_gpu_layers = 99, n_threads = 36, n_threads_batch = 36 | PP | TG | N_KV | T_PP s | S_PP t/s | T_TG s | S_TG t/s | |-------|--------|--------|----------|----------|----------|----------| | 512 | 128 | 0 | 25.871 | 19.79 | 27.952 | 4.58 | | 512 | 128 | 512 | 25.531 | 20.05 | 27.689 | 4.62 | --- # --threads 36 + -ub 1024 -b 1024 CUDA_DEVICE_ORDER=PCI_BUS_ID \ CUDA_VISIBLE_DEVICES=0 \ ~/ik_llama-main-b3746-f26fe36-bin-win-cuda-12.8-x64/llama-sweep-bench \ -m DeepSeek-R1-0528-IQ1_S_R4-00001-of-00003.gguf \ -c 8192 \ -mla 3 -fa \ -amb 512 \ -fmoe \ -ngl 99 \ -ot exps=CPU \ --warmup-batch \ --threads 36 \ -ub 1024 -b 1024 --- main: n_kv_max = 8192, n_batch = 1024, n_ubatch = 1024, flash_attn = 1, n_gpu_layers = 99, n_threads = 36, n_threads_batch = 36 | PP | TG | N_KV | T_PP s | S_PP t/s | T_TG s | S_TG t/s | |-------|--------|--------|----------|----------|----------|----------| | 1024 | 256 | 0 | 47.999 | 21.33 | 55.530 | 4.61 | | 1024 | 256 | 1024 | 47.737 | 21.45 | 55.263 | 4.63 | --- # --threads 36 + -ub 2028 -b 2048 CUDA_DEVICE_ORDER=PCI_BUS_ID \ CUDA_VISIBLE_DEVICES=0 \ ~/ik_llama-main-b3746-f26fe36-bin-win-cuda-12.8-x64/llama-sweep-bench \ -m DeepSeek-R1-0528-IQ1_S_R4-00001-of-00003.gguf \ -c 8192 \ -mla 3 -fa \ -amb 512 \ -fmoe \ -ngl 99 \ -ot exps=CPU \ --warmup-batch \ --threads 36 \ -ub 2028 -b 2048 --- main: n_kv_max = 8192, n_batch = 2048, n_ubatch = 2028, flash_attn = 1, n_gpu_layers = 99, n_threads = 36, n_threads_batch = 36 | PP | TG | N_KV | T_PP s | S_PP t/s | T_TG s | S_TG t/s | |-------|--------|--------|----------|----------|----------|----------| | 2028 | 507 | 0 | 94.077 | 21.56 | 109.357 | 4.64 | | 2028 | 507 | 2028 | 101.274 | 20.02 | 131.667 | 3.85 | --- ## Other benches # BENCH IT! CUDA_DEVICE_ORDER=PCI_BUS_ID CUDA_VISIBLE_DEVICES=0,2,1 ~/ik_llama-main-b3746-f26fe36-bin-win-cuda-12.8-x64/llama-bench -m DeepSeek-R1-0528-IQ1_S_R4-00001-of-00003.gguf -fmoe 1 -n 0 --override-tensor exps=CPU -ngl 62 -fa --warmup-batch --main-gpu 0 --override-tensor exps=CPU -t 36 -mla 3 -b 128,512,1024,2048,4096 -ub 128,512,1024,2048,4096 -rtr 1 ggml_cuda_init: GGML_CUDA_FORCE_MMQ: no ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no ggml_cuda_init: found 3 CUDA devices: Device 0: NVIDIA GeForce RTX 5090, compute capability 12.0, VMM: yes Device 1: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes Device 2: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes | model | size | params | backend | ngl | threads | n_batch | n_ubatch | mla | rtr | fmoe | test | t/s | | ------------------------------ | ---------: | ---------: | ---------- | --: | ------: | ------: | -------: | --: | --: | ---: | ------------: | ---------------: | | deepseek2 671B IQ1_S_R4 - 1.5 bpw | 130.20 GiB | 672.05 B | CUDA | 62 | 36 | 128 | 128 | 3 | 1 | 1 | pp512 | 16.25 ± 0.04 | | deepseek2 671B IQ1_S_R4 - 1.5 bpw | 130.20 GiB | 672.05 B | CUDA | 62 | 36 | 128 | 512 | 3 | 1 | 1 | pp512 | 16.28 ± 0.05 | | deepseek2 671B IQ1_S_R4 - 1.5 bpw | 130.20 GiB | 672.05 B | CUDA | 62 | 36 | 128 | 1024 | 3 | 1 | 1 | pp512 | 16.26 ± 0.07 | | deepseek2 671B IQ1_S_R4 - 1.5 bpw | 130.20 GiB | 672.05 B | CUDA | 62 | 36 | 128 | 2048 | 3 | 1 | 1 | pp512 | 16.24 ± 0.14 | | deepseek2 671B IQ1_S_R4 - 1.5 bpw | 130.20 GiB | 672.05 B | CUDA | 62 | 36 | 128 | 4096 | 3 | 1 | 1 | pp512 | 16.19 ± 0.20 | | deepseek2 671B IQ1_S_R4 - 1.5 bpw | 130.20 GiB | 672.05 B | CUDA | 62 | 36 | 512 | 128 | 3 | 1 | 1 | pp512 | 16.23 ± 0.14 | | deepseek2 671B IQ1_S_R4 - 1.5 bpw | 130.20 GiB | 672.05 B | CUDA | 62 | 36 | 512 | 512 | 3 | 1 | 1 | pp512 | 20.27 ± 0.10 | | deepseek2 671B IQ1_S_R4 - 1.5 bpw | 130.20 GiB | 672.05 B | CUDA | 62 | 36 | 512 | 1024 | 3 | 1 | 1 | pp512 | 20.24 ± 0.10 | | deepseek2 671B IQ1_S_R4 - 1.5 bpw | 130.20 GiB | 672.05 B | CUDA | 62 | 36 | 512 | 2048 | 3 | 1 | 1 | pp512 | 20.22 ± 0.12 | | deepseek2 671B IQ1_S_R4 - 1.5 bpw | 130.20 GiB | 672.05 B | CUDA | 62 | 36 | 512 | 4096 | 3 | 1 | 1 | pp512 | 20.30 ± 0.04 | | deepseek2 671B IQ1_S_R4 - 1.5 bpw | 130.20 GiB | 672.05 B | CUDA | 62 | 36 | 1024 | 128 | 3 | 1 | 1 | pp512 | 16.19 ± 0.08 | | deepseek2 671B IQ1_S_R4 - 1.5 bpw | 130.20 GiB | 672.05 B | CUDA | 62 | 36 | 1024 | 512 | 3 | 1 | 1 | pp512 | 20.10 ± 0.12 | | deepseek2 671B IQ1_S_R4 - 1.5 bpw | 130.20 GiB | 672.05 B | CUDA | 62 | 36 | 1024 | 1024 | 3 | 1 | 1 | pp512 | 20.30 ± 0.03 | | deepseek2 671B IQ1_S_R4 - 1.5 bpw | 130.20 GiB | 672.05 B | CUDA | 62 | 36 | 1024 | 2048 | 3 | 1 | 1 | pp512 | 20.29 ± 0.04 | | deepseek2 671B IQ1_S_R4 - 1.5 bpw | 130.20 GiB | 672.05 B | CUDA | 62 | 36 | 1024 | 4096 | 3 | 1 | 1 | pp512 | 20.15 ± 0.09 | | deepseek2 671B IQ1_S_R4 - 1.5 bpw | 130.20 GiB | 672.05 B | CUDA | 62 | 36 | 2048 | 128 | 3 | 1 | 1 | pp512 | 16.17 ± 0.15 | | deepseek2 671B IQ1_S_R4 - 1.5 bpw | 130.20 GiB | 672.05 B | CUDA | 62 | 36 | 2048 | 512 | 3 | 1 | 1 | pp512 | 20.21 ± 0.10 | | deepseek2 671B IQ1_S_R4 - 1.5 bpw | 130.20 GiB | 672.05 B | CUDA | 62 | 36 | 2048 | 1024 | 3 | 1 | 1 | pp512 | 20.17 ± 0.11 | | deepseek2 671B IQ1_S_R4 - 1.5 bpw | 130.20 GiB | 672.05 B | CUDA | 62 | 36 | 2048 | 2048 | 3 | 1 | 1 | pp512 | 20.24 ± 0.10 | | deepseek2 671B IQ1_S_R4 - 1.5 bpw | 130.20 GiB | 672.05 B | CUDA | 62 | 36 | 2048 | 4096 | 3 | 1 | 1 | pp512 | 20.23 ± 0.12 | | deepseek2 671B IQ1_S_R4 - 1.5 bpw | 130.20 GiB | 672.05 B | CUDA | 62 | 36 | 4096 | 128 | 3 | 1 | 1 | pp512 | 16.22 ± 0.10 | | deepseek2 671B IQ1_S_R4 - 1.5 bpw | 130.20 GiB | 672.05 B | CUDA | 62 | 36 | 4096 | 512 | 3 | 1 | 1 | pp512 | 20.13 ± 0.09 | | deepseek2 671B IQ1_S_R4 - 1.5 bpw | 130.20 GiB | 672.05 B | CUDA | 62 | 36 | 4096 | 1024 | 3 | 1 | 1 | pp512 | 20.30 ± 0.04 | | deepseek2 671B IQ1_S_R4 - 1.5 bpw | 130.20 GiB | 672.05 B | CUDA | 62 | 36 | 4096 | 2048 | 3 | 1 | 1 | pp512 | 20.05 ± 0.21 | | deepseek2 671B IQ1_S_R4 - 1.5 bpw | 130.20 GiB | 672.05 B | CUDA | 62 | 36 | 4096 | 4096 | 3 | 1 | 1 | pp512 | 20.21 ± 0.07 | build: f26fe36 (1) # BENCH IT! CUDA_DEVICE_ORDER=PCI_BUS_ID CUDA_VISIBLE_DEVICES=0,2,1 ~/ik_llama-main-b3746-f26fe36-bin-win-cuda-12.8-x64/llama-bench -m DeepSeek-R1-0528-IQ1_S_R4-00001-of-00003.gguf -fmoe 1 -p 0 -n 60 --override-tensor exps=CPU -ngl 62 -fa --warmup-batch --main-gpu 0 --override-tensor exps=CPU -t 36 -mla 3 -b 128,512,1024,2048,4096 -ub 128,512,1024,2048,4096 -rtr 1 ggml_cuda_init: GGML_CUDA_FORCE_MMQ: no ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no ggml_cuda_init: found 3 CUDA devices: Device 0: NVIDIA GeForce RTX 5090, compute capability 12.0, VMM: yes Device 1: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes Device 2: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes | model | size | params | backend | ngl | threads | n_batch | n_ubatch | mla | rtr | fmoe | test | t/s | | ------------------------------ | ---------: | ---------: | ---------- | --: | ------: | ------: | -------: | --: | --: | ---: | ------------: | ---------------: | | deepseek2 671B IQ1_S_R4 - 1.5 bpw | 130.20 GiB | 672.05 B | CUDA | 62 | 36 | 128 | 128 | 3 | 1 | 1 | tg60 | 3.18 ± 0.03 | | deepseek2 671B IQ1_S_R4 - 1.5 bpw | 130.20 GiB | 672.05 B | CUDA | 62 | 36 | 128 | 512 | 3 | 1 | 1 | tg60 | 3.17 ± 0.03 | | deepseek2 671B IQ1_S_R4 - 1.5 bpw | 130.20 GiB | 672.05 B | CUDA | 62 | 36 | 128 | 1024 | 3 | 1 | 1 | tg60 | 3.17 ± 0.04 | | deepseek2 671B IQ1_S_R4 - 1.5 bpw | 130.20 GiB | 672.05 B | CUDA | 62 | 36 | 128 | 2048 | 3 | 1 | 1 | tg60 | 3.14 ± 0.02 | | deepseek2 671B IQ1_S_R4 - 1.5 bpw | 130.20 GiB | 672.05 B | CUDA | 62 | 36 | 128 | 4096 | 3 | 1 | 1 | tg60 | 3.17 ± 0.05 | | deepseek2 671B IQ1_S_R4 - 1.5 bpw | 130.20 GiB | 672.05 B | CUDA | 62 | 36 | 512 | 128 | 3 | 1 | 1 | tg60 | 3.14 ± 0.03 | | deepseek2 671B IQ1_S_R4 - 1.5 bpw | 130.20 GiB | 672.05 B | CUDA | 62 | 36 | 512 | 512 | 3 | 1 | 1 | tg60 | 3.17 ± 0.03 | | deepseek2 671B IQ1_S_R4 - 1.5 bpw | 130.20 GiB | 672.05 B | CUDA | 62 | 36 | 512 | 1024 | 3 | 1 | 1 | tg60 | 3.14 ± 0.03 | | deepseek2 671B IQ1_S_R4 - 1.5 bpw | 130.20 GiB | 672.05 B | CUDA | 62 | 36 | 512 | 2048 | 3 | 1 | 1 | tg60 | 3.17 ± 0.02 | | deepseek2 671B IQ1_S_R4 - 1.5 bpw | 130.20 GiB | 672.05 B | CUDA | 62 | 36 | 512 | 4096 | 3 | 1 | 1 | tg60 | 3.15 ± 0.01 | | deepseek2 671B IQ1_S_R4 - 1.5 bpw | 130.20 GiB | 672.05 B | CUDA | 62 | 36 | 1024 | 128 | 3 | 1 | 1 | tg60 | 3.18 ± 0.03 | | deepseek2 671B IQ1_S_R4 - 1.5 bpw | 130.20 GiB | 672.05 B | CUDA | 62 | 36 | 1024 | 512 | 3 | 1 | 1 | tg60 | 3.17 ± 0.02 | | deepseek2 671B IQ1_S_R4 - 1.5 bpw | 130.20 GiB | 672.05 B | CUDA | 62 | 36 | 1024 | 1024 | 3 | 1 | 1 | tg60 | 3.17 ± 0.03 | | deepseek2 671B IQ1_S_R4 - 1.5 bpw | 130.20 GiB | 672.05 B | CUDA | 62 | 36 | 1024 | 2048 | 3 | 1 | 1 | tg60 | 3.15 ± 0.02 | | deepseek2 671B IQ1_S_R4 - 1.5 bpw | 130.20 GiB | 672.05 B | CUDA | 62 | 36 | 1024 | 4096 | 3 | 1 | 1 | tg60 | 3.14 ± 0.04 | | deepseek2 671B IQ1_S_R4 - 1.5 bpw | 130.20 GiB | 672.05 B | CUDA | 62 | 36 | 2048 | 128 | 3 | 1 | 1 | tg60 | 3.15 ± 0.02 | | deepseek2 671B IQ1_S_R4 - 1.5 bpw | 130.20 GiB | 672.05 B | CUDA | 62 | 36 | 2048 | 512 | 3 | 1 | 1 | tg60 | 3.18 ± 0.02 | | deepseek2 671B IQ1_S_R4 - 1.5 bpw | 130.20 GiB | 672.05 B | CUDA | 62 | 36 | 2048 | 1024 | 3 | 1 | 1 | tg60 | 3.16 ± 0.04 | | deepseek2 671B IQ1_S_R4 - 1.5 bpw | 130.20 GiB | 672.05 B | CUDA | 62 | 36 | 2048 | 2048 | 3 | 1 | 1 | tg60 | 3.17 ± 0.03 | | deepseek2 671B IQ1_S_R4 - 1.5 bpw | 130.20 GiB | 672.05 B | CUDA | 62 | 36 | 2048 | 4096 | 3 | 1 | 1 | tg60 | 3.16 ± 0.02 | | deepseek2 671B IQ1_S_R4 - 1.5 bpw | 130.20 GiB | 672.05 B | CUDA | 62 | 36 | 4096 | 128 | 3 | 1 | 1 | tg60 | 3.17 ± 0.04 | | deepseek2 671B IQ1_S_R4 - 1.5 bpw | 130.20 GiB | 672.05 B | CUDA | 62 | 36 | 4096 | 512 | 3 | 1 | 1 | tg60 | 3.14 ± 0.03 | | deepseek2 671B IQ1_S_R4 - 1.5 bpw | 130.20 GiB | 672.05 B | CUDA | 62 | 36 | 4096 | 1024 | 3 | 1 | 1 | tg60 | 3.17 ± 0.02 | | deepseek2 671B IQ1_S_R4 - 1.5 bpw | 130.20 GiB | 672.05 B | CUDA | 62 | 36 | 4096 | 2048 | 3 | 1 | 1 | tg60 | 2.95 ± 0.23 | | deepseek2 671B IQ1_S_R4 - 1.5 bpw | 130.20 GiB | 672.05 B | CUDA | 62 | 36 | 4096 | 4096 | 3 | 1 | 1 | tg60 | 3.17 ± 0.02 | build: f26fe36 (1)