logo

InferX AI Function Platform (Lambda Function for Inference)

    --   Serve tens models in one box with ultra-fast (<2 sec) cold start (contact: support@inferx.net)

Model Aquila-7B

namespace model name standby gpu standby pageable standby pinned memory gpu count vRam (MB) cpu memory (MB) state revision
BAAI Aquila-7B Blob Blob Blob 2 10000 20.0 50000 Normal 426

Image

Prompt



Sample Rest Call

Pods

tenant namespace pod name state require resource allocated resource
public BAAI public/BAAI/Aquila-7B/426/915 Standby {'CPU': 20000, 'Mem': 50000, 'GPU': {'Type': 'Any', 'Count': 2, 'vRam': 10000}} {'nodename': 'node3', 'CPU': 20000, 'Mem': 50000, 'GPUType': 'A4000', 'GPUs': {'vRam': 0, 'map': {}, 'slotSize': 0, 'totalSlotCnt': 0}, 'MaxContextPerGPU': 2}

Failures

tenant namespace model name revision id exit info state
public BAAI Aquila-7B 426 599 Error("DockerResponseServerError { status_code: 404, message: \"No such container: public.BAAI.Aquila-7B.426.599\" }") log
public BAAI Aquila-7B 426 661 Error("DockerContainerWaitError { error: \"\", code: 139 }") log

Func

{
 "image": "vllm/vllm-openai:v0.6.2",
 "commands": [
  "--model",
  "BAAI/Aquila-7B",
  "--enforce-eager",
  "--disable-custom-all-reduce",
  "--trust-remote-code",
  "--max-model-len",
  "2000",
  "--tensor-parallel-size=2"
 ],
 "envs": [
  [
   "LD_LIBRARY_PATH",
   "/usr/local/lib/python3.12/dist-packages/nvidia/cuda_nvrtc/lib/:$LD_LIBRARY_PATH"
  ]
 ],
 "mounts": [
  {
   "hostpath": "/home/brad/cache",
   "mountpath": "/root/.cache/huggingface"
  }
 ],
 "endpoint": {
  "port": 8000,
  "schema": "Http",
  "probe": "/health"
 },
 "version": 426,
 "entrypoint": [],
 "resources": {
  "CPU": 20000,
  "Mem": 50000,
  "GPU": {
   "Type": "Any",
   "Count": 2,
   "vRam": 10000
  }
 },
 "standby": {
  "gpu": "Blob",
  "pageable": "Blob",
  "pinned": "Blob"
 },
 "probe": {
  "port": 80,
  "schema": "Http",
  "probe": "/health"
 },
 "sample_query": {
  "apiType": "openai",
  "path": "v1/completions",
  "prompt": "\u6c7d\u8f66EDR\u662f\u4ec0\u4e48",
  "body": {
   "max_tokens": "1000",
   "model": "BAAI/Aquila-7B",
   "stream": "true",
   "temperature": "0"
  }
 }
}