Speculative Decoding with SGLang

This page is for version dev. For other versions, use one of these documentation indexes: - Latest (v1.5.1) (default): https://docs.nvidia.com/dynamo/latest/llms.txt - dev: https://docs.nvidia.com/dynamo/dev/llms.txt - v1.5.1: https://docs.nvidia.com/dynamo/v1.5.1/llms.txt - v1.5.0: https://docs.nvidia.com/dynamo/v1.5.0/llms.txt - v1.4.2: https://docs.nvidia.com/dynamo/v1.4.2/llms.txt - v1.4.1: https://docs.nvidia.com/dynamo/v1.4.1/llms.txt - v1.4.0: https://docs.nvidia.com/dynamo/v1.4.0/llms.txt - v1.3.0: https://docs.nvidia.com/dynamo/v1.3.0/llms.txt - v1.2.1: https://docs.nvidia.com/dynamo/v1.2.1/llms.txt - v1.2.0: https://docs.nvidia.com/dynamo/v1.2.0/llms.txt - v1.1.1: https://docs.nvidia.com/dynamo/v1.1.1/llms.txt - v1.1.0: https://docs.nvidia.com/dynamo/v1.1.0/llms.txt - v1.0.2: https://docs.nvidia.com/dynamo/v1.0.2/llms.txt - v1.0.1: https://docs.nvidia.com/dynamo/v1.0.1/llms.txt - v1.0.0: https://docs.nvidia.com/dynamo/v1.0.0/llms.txt - v0.9.1: https://docs.nvidia.com/dynamo/v-0-9-1/llms.txt - v0.9.0: https://docs.nvidia.com/dynamo/v-0-9-0/llms.txt - v0.8.1: https://docs.nvidia.com/dynamo/v-0-8-1/llms.txt - v0.8.0: https://docs.nvidia.com/dynamo/v-0-8-0/llms.txt - v0.7.1: https://docs.nvidia.com/dynamo/v-0-7-1/llms.txt - v0.7.0: https://docs.nvidia.com/dynamo/v-0-7-0/llms.txt

docker compose -f dev/docker-compose.yml up -d

cd examples/backends/sglang
./launch/agg_spec_decoding.sh

curl http://localhost:8000/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "Qwen/Qwen3-8B",
    "messages": [
      {"role": "user", "content": "Write a poem about why Sakura trees are beautiful."}
    ],
    "max_tokens": 250
  }'

curl -s localhost:8081/metrics | grep '^sglang:spec_'

python3 -m dynamo.frontend --router-mode kv &
SPEC_ARGS=(
  --model-path Qwen/Qwen3-8B
  --page-size 16
  --enable-metrics
  --speculative-algorithm EAGLE3
  --speculative-draft-model-path Tengyunw/qwen3_8b_eagle3
  --speculative-num-steps 3
  --speculative-eagle-topk 1
  --speculative-num-draft-tokens 4
  --disaggregation-transfer-backend nixl
  --host 0.0.0.0
)
DYN_SYSTEM_PORT=8081 CUDA_VISIBLE_DEVICES=0 python3 -m dynamo.sglang "${SPEC_ARGS[@]}" \
  --disaggregation-mode prefill \
  --kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' &
DYN_SYSTEM_PORT=8082 CUDA_VISIBLE_DEVICES=1 python3 -m dynamo.sglang "${SPEC_ARGS[@]}" \
  --disaggregation-mode decode \
  --kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5558"}' &