GGL Docs

inference-84

2 min read

Access

cat << EOF >> ~/.ssh/config
Host inference-84 
    HostName inference-84.astral-labs.work 
    User sysadmin
    IdentityFile ~/.ssh/id_rsa
    ProxyCommand /opt/homebrew/bin/cloudflared access ssh --hostname %h
EOF

ssh inference-84

Config

  • vllm
  • litellm
  • openwebUI

Dependencies

sudo ubuntu-drivers list
 
sudo ubuntu-drivers install
 
sudo reboot
 
sudo apt install nvidia-cuda-toolkit
 
# Install container deps
# https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html
sudo apt-get update && sudo apt-get install -y --no-install-recommends \
       ca-certificates \
       curl \
       gnupg2
       
       
curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg \
  && curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | \
    sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | \
    sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list
    
    
sudo apt-get update
 
export NVIDIA_CONTAINER_TOOLKIT_VERSION=1.19.1-1
sudo apt-get install -y \
      nvidia-container-toolkit=${NVIDIA_CONTAINER_TOOLKIT_VERSION} \
      nvidia-container-toolkit-base=${NVIDIA_CONTAINER_TOOLKIT_VERSION} \
      libnvidia-container-tools=${NVIDIA_CONTAINER_TOOLKIT_VERSION} \
      libnvidia-container1=${NVIDIA_CONTAINER_TOOLKIT_VERSION}
      
sudo nvidia-ctk runtime configure --runtime=docker
 
cat /etc/docker/daemon.json
{
    "runtimes": {
        "nvidia": {
            "args": [],
            "path": "nvidia-container-runtime"
        }
    }
}
 
sudo systemctl restart docker
   
docker run -it --rm --gpus device=0 ubuntu nvidia-smi

Compose

services:
  vllm:
    image: vllm/vllm-openai:latest
    restart: unless-stopped
    ports:
      - "127.0.0.1:8000:8000"
    volumes:
      - huggingface_cache:/root/.cache/huggingface
    ipc: host
    entrypoint: vllm serve
    # entrypoint: vllm bench throughput
    command: >
      --model Qwen/Qwen3.6-27B-FP8
      --tensor-parallel-size 1
      --pipeline-parallel-size 3
      --quantization fp8
      --max-model-len 131072
      --gpu-memory-utilization 0.80
      --enforce-eager
      --enable-auto-tool-choice
      --tool-call-parser qwen3_coder
      --reasoning-parser qwen3
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: 3
              capabilities: [gpu]
  # ollama:
  #   image: ollama/ollama:latest
  #   restart: unless-stopped
  #   ports:
  #     - "127.0.0.1:11434:11434"
  #   volumes:
  #     - ollama_data:/root/.ollama
  #   deploy:
  #     resources:
  #       reservations:
  #         devices:
  #           - driver: nvidia
  #             count: 3
  #             capabilities: [gpu]
 
  litellm:
    image: ghcr.io/berriai/litellm:main-stable@sha256:66a108711edea25ef531a74764f001d0c1934cc8abb422f5a3bd17a2860e4035
    restart: unless-stopped
    ports:
      - "0.0.0.0:4000:4000"
    volumes:
      - ./litellm_config.yaml:/app/config.yaml
    command: --config /app/config.yaml --port 4000
    environment:
      - DATABASE_URL=postgresql://litellm:litellm@db:5432/litellm
    depends_on:
      - vllm
      - db
 
  db:
    image: postgres:16
    restart: unless-stopped
    environment:
      POSTGRES_USER: litellm
      POSTGRES_PASSWORD: litellm
      POSTGRES_DB: litellm
    volumes:
      - postgres_data:/var/lib/postgresql/data
 
volumes:
  huggingface_cache:
  ollama_data:
  postgres_data:
 
 
# unsloth/Qwen3.6-35B-A3B-MTP-GGUF
 
 
# --model Qwen/Qwen3.6-35B-FP8
# --tensor-parallel-size 1
# --pipeline-parallel-size 3
# --gpu-memory-utilization 0.95

Connect Claude

curl -fsSL https://claude.ai/install.sh | bash
 
# Create a virtual key in LiteLLM - http://10.10.12.84:4000/ui/?page=api-keys
export ANTHROPIC_AUTH_TOKEN="$LITELLM_VIRTUAL_KEY"
export ANTHROPIC_BASE_URL="http://10.10.12.84:4000"
export CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY=1
 
claude --model Qwen3.6-27B-FP8

Graph View