llama-cpp

# Download from HuggingFace (GGUF format)
huggingface-cli download \
    TheBloke/Llama-2-7B-Chat-GGUF \
    llama-2-7b-chat.Q4_K_M.gguf \
    --local-dir models/
 
# Or convert from HuggingFace
python convert_hf_to_gguf.py models/llama-2-7b-chat/

# Simple chat
./llama-cli \
    -m models/llama-2-7b-chat.Q4_K_M.gguf \
    -p "Explain quantum computing" \
    -n 256  # Max tokens
 
# Interactive chat
./llama-cli \
    -m models/llama-2-7b-chat.Q4_K_M.gguf \
    --interactive

# Start OpenAI-compatible server
./llama-server \
    -m models/llama-2-7b-chat.Q4_K_M.gguf \
    --host 0.0.0.0 \
    --port 8080 \
    -ngl 32  # Offload 32 layers to GPU
 
# Client request
curl http://localhost:8080/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "llama-2-7b-chat",
    "messages": [{"role": "user", "content": "Hello!"}],
    "temperature": 0.7,
    "max_tokens": 100
  }'

# General use (balanced)
Q4_K_M  # 4-bit, medium quality
 
# Maximum speed (more degradation)
Q2_K or Q3_K_M
 
# Maximum quality (slower)
Q6_K or Q8_0
 
# Very large models (70B, 405B)
Q3_K_M or Q4_K_S  # Lower bits to fit in memory

# Build with Metal
make LLAMA_METAL=1
 
# Run with GPU acceleration (automatic)
./llama-cli -m model.gguf -ngl 999  # Offload all layers
 
# Performance: M3 Max 40-60 tokens/sec (Llama 2-7B Q4_K_M)

# Build with CUDA
make LLAMA_CUDA=1
 
# Offload layers to GPU
./llama-cli -m model.gguf -ngl 35  # Offload 35/40 layers
 
# Hybrid CPU+GPU for large models
./llama-cli -m llama-70b.Q4_K_M.gguf -ngl 20  # GPU: 20 layers, CPU: rest

# Build with ROCm
make LLAMA_HIP=1
 
# Run with AMD GPU
./llama-cli -m model.gguf -ngl 999

# Process multiple prompts from file
cat prompts.txt | ./llama-cli \
    -m model.gguf \
    --batch-size 512 \
    -n 100

# JSON output with grammar
./llama-cli \
    -m model.gguf \
    -p "Generate a person: " \
    --grammar-file grammars/json.gbnf
 
# Outputs valid JSON only

# Increase context (default 512)
./llama-cli \
    -m model.gguf \
    -c 4096  # 4K context window
 
# Very long context (if model supports)
./llama-cli -m model.gguf -c 32768  # 32K context