logo

InferX AI Function Platform (Lambda Function for Inference)

    --   Serve tens models in one box with ultra-fast (<2 sec) cold start (contact: support@inferx.net)

Model Qwen2.5-7B-Instruct-GPTQ-Int8

namespace model name standby gpu standby pageable standby pinned memory gpu count vRam (MB) cpu memory (MB) state revision
Qwen Qwen2.5-7B-Instruct-GPTQ-Int8 Blob Blob Blob 1 13500 20.0 30000 Normal 126

Image

Prompt



Sample Rest Call

Pods

tenant namespace pod name state require resource allocated resource
public Qwen public/Qwen/Qwen2.5-7B-Instruct-GPTQ-Int8/126/965 Standby {'CPU': 20000, 'Mem': 30000, 'GPU': {'Type': 'Any', 'Count': 1, 'vRam': 13500}} {'nodename': 'node3', 'CPU': 20000, 'Mem': 30000, 'GPUType': 'A4000', 'GPUs': {'vRam': 0, 'map': {}, 'slotSize': 0, 'totalSlotCnt': 0}, 'MaxContextPerGPU': 2}

Func

{
 "image": "vllm/vllm-openai:v0.6.2",
 "commands": [
  "--model",
  "Qwen/Qwen2.5-7B-Instruct-GPTQ-Int8",
  "--enforce-eager",
  "--gpu-memory-utilization",
  "0.99",
  "--max-model-len",
  "1000"
 ],
 "envs": [
  [
   "LD_LIBRARY_PATH",
   "/usr/local/lib/python3.12/dist-packages/nvidia/cuda_nvrtc/lib/:$LD_LIBRARY_PATH"
  ]
 ],
 "mounts": [
  {
   "hostpath": "/home/brad/cache",
   "mountpath": "/root/.cache/huggingface"
  }
 ],
 "endpoint": {
  "port": 8000,
  "schema": "Http",
  "probe": "/health"
 },
 "version": 126,
 "entrypoint": [],
 "resources": {
  "CPU": 20000,
  "Mem": 30000,
  "GPU": {
   "Type": "Any",
   "Count": 1,
   "vRam": 13500
  }
 },
 "standby": {
  "gpu": "Blob",
  "pageable": "Blob",
  "pinned": "Blob"
 },
 "probe": {
  "port": 80,
  "schema": "Http",
  "probe": "/health"
 },
 "sample_query": {
  "apiType": "openai",
  "path": "v1/completions",
  "prompt": "Give me a short introduction to large language model.",
  "body": {
   "max_tokens": "800",
   "model": "Qwen/Qwen2.5-7B-Instruct-GPTQ-Int8",
   "stream": "true",
   "temperature": "0"
  }
 }
}