vllm-serving

[project]
name = "vllm-serving"
version = "0.1.0"
description = "vLLM serving environment"
requires-python = "==3.12.*"
dependencies = [
  "torch==2.9.1; platform_system == 'Linux' and platform_machine == 'x86_64'",
  "torchvision==0.24.1; platform_system == 'Linux' and platform_machine == 'x86_64'",
  "torchaudio==2.9.1; platform_system == 'Linux' and platform_machine == 'x86_64'",
  "torch==2.10.0; platform_system != 'Linux' or platform_machine != 'x86_64'",
  "torchvision==0.25.0; platform_system != 'Linux' or platform_machine != 'x86_64'",
  "torchaudio==2.10.0; platform_system != 'Linux' or platform_machine != 'x86_64'",
  "vllm==0.15.1; platform_system == 'Linux' and platform_machine == 'x86_64'",
  "flashinfer-python==0.6.1; platform_system == 'Linux' and platform_machine == 'x86_64'",
  "xgrammar==0.1.29",
  "transformers==4.57.6",
  "openai==2.21.0",
  "ray==2.54.0",
  "ipykernel==7.2.0",
  "pytest==9.0.2",
  "ruff==0.15.2",
]

[tool.uv]
package = false

uv run python -c "import torch, transformers, openai, xgrammar; print('torch', torch.__version__); print('transformers', transformers.__version__); print('xgrammar', xgrammar.__version__)"

uv run python -c "import vllm; print('vllm', vllm.__version__)"

uv run python -m vllm.entrypoints.openai.api_server \
  --model meta-llama/Llama-3.1-8B-Instruct \
  --host 0.0.0.0 \
  --port 8000 \
  --max-model-len 4096

from openai import OpenAI

client = OpenAI(
    base_url="http://localhost:8000/v1",
    api_key="token-not-needed"  # vLLM 자체 서빙 시 인증 불필요
)

response = client.chat.completions.create(
    model="meta-llama/Llama-3.1-8B-Instruct",
    messages=[{"role": "user", "content": "한국의 수도는?"}],
    max_tokens=256
)
print(response.choices[0].message.content)