[{"data":1,"prerenderedAt":771},["ShallowReactive",2],{"page-\u002Fexamples\u002Fvllm-server":3,"surround-\u002Fexamples\u002Fvllm-server":762},{"id":4,"title":5,"body":6,"description":754,"extension":755,"meta":756,"navigation":757,"path":758,"seo":759,"stem":760,"__hash__":761},"docs\u002Fexamples\u002Fvllm-server.md","Serve an LLM with vLLM",{"type":7,"value":8,"toc":745},"minimark",[9,18,23,34,38,120,124,406,414,418,425,444,450,454,468,549,552,673,682,686,689,717,721,741],[10,11,12,13,17],"p",{},"This example deploys the vLLM template on one GPU, serves ",[14,15,16],"code",{},"Qwen\u002FQwen2.5-7B-Instruct"," at a public HTTPS address with an API key, and calls it with the OpenAI chat completions API. The model files are cached on a network volume, so later starts skip the download.",[19,20,22],"h2",{"id":21},"what-you-need","What you need",[24,25,26],"ul",{},[27,28,29,30,33],"li",{},"An API key for your server, for example from ",[14,31,32],{},"openssl rand -hex 24",". Clients send it to call the model.",[19,35,37],{"id":36},"step-1-create-a-network-volume-for-the-model-cache","Step 1: Create a network volume for the model cache",[39,40,42,79],"tabs",{"sync":41},"interface",[43,44,46],"tabs-item",{"label":45},"Console",[10,47,48,49,58,59,63,64,67,68,71,72,75,76,78],{},"Open the ",[50,51,57],"a",{"href":52,"rel":53,"target":56},"https:\u002F\u002Fconsole.cloud.featherless.ai\u002Fstorage",[54,55],"noopener","noreferrer","_blank","Storage page",", click ",[60,61,62],"strong",{},"Create volume",", name it ",[14,65,66],{},"models",", set ",[60,69,70],{},"Capacity"," to ",[14,73,74],{},"100",", and click ",[60,77,62],{},".",[43,80,82],{"label":81},"CLI",[83,84,89],"pre",{"className":85,"code":86,"language":87,"meta":88,"style":88},"language-bash shiki shiki-themes github-dark-default","featherless-cloud volume create models --gb 100 --wait\n","bash","",[14,90,91],{"__ignoreMap":88},[92,93,96,100,104,107,110,114,117],"span",{"class":94,"line":95},"line",1,[92,97,99],{"class":98},"sQhOw","featherless-cloud",[92,101,103],{"class":102},"s9uIt"," volume",[92,105,106],{"class":102}," create",[92,108,109],{"class":102}," models",[92,111,113],{"class":112},"sFSAA"," --gb",[92,115,116],{"class":112}," 100",[92,118,119],{"class":112}," --wait\n",[19,121,123],{"id":122},"step-2-deploy-the-server","Step 2: Deploy the server",[39,125,126,294],{"sync":41},[43,127,128],{"label":45},[129,130,132,137,164,168,182,186,201,205,214,218,224,232,238,280,284],"steps",{"level":131},"4",[133,134,136],"h4",{"id":135},"choose-the-template","Choose the template",[10,138,48,139,144,145,148,149,152,153,156,157,160,161,78],{},[50,140,143],{"href":141,"rel":142,"target":56},"https:\u002F\u002Fconsole.cloud.featherless.ai\u002Finstances",[54,55],"Instances page"," and click ",[60,146,147],{},"Deploy instance",". Name the instance ",[14,150,151],{},"vllm",". Under ",[60,154,155],{},"Template",", search for ",[60,158,159],{},"vLLM"," and select ",[60,162,163],{},"vLLM (ROCm)",[133,165,167],{"id":166},"choose-compute","Choose compute",[10,169,170,171,174,175,178,179,78],{},"Under ",[60,172,173],{},"Compute",", choose a ",[60,176,177],{},"GPU model"," and ",[60,180,181],{},"1 GPU",[133,183,185],{"id":184},"mount-the-volume","Mount the volume",[10,187,170,188,58,191,194,195,197,198,78],{},[60,189,190],{},"Network volumes",[60,192,193],{},"Add network volume",", choose ",[14,196,66],{},", and keep the mount path ",[14,199,200],{},"\u002Fworkspace",[133,202,204],{"id":203},"expose-the-api-port","Expose the API port",[10,206,170,207,210,211,78],{},[60,208,209],{},"Public ports",", enter ",[14,212,213],{},"8000",[133,215,217],{"id":216},"start-vllm","Start vLLM",[10,219,170,220,223],{},[60,221,222],{},"Start command",", enter:",[83,225,230],{"className":226,"code":228,"language":229,"meta":88},[227],"language-text","vllm serve Qwen\u002FQwen2.5-7B-Instruct --host 0.0.0.0 --port 8000\n","text",[14,231,228],{"__ignoreMap":88},[10,233,170,234,237],{},[60,235,236],{},"Environment variables",", add:",[239,240,241,254],"table",{},[242,243,244],"thead",{},[245,246,247,251],"tr",{},[248,249,250],"th",{},"Name",[248,252,253],{},"Value",[255,256,257,270],"tbody",{},[245,258,259,265],{},[260,261,262],"td",{},[14,263,264],{},"HF_HOME",[260,266,267],{},[14,268,269],{},"\u002Fworkspace\u002Fhf",[245,271,272,277],{},[260,273,274],{},[14,275,276],{},"VLLM_API_KEY",[260,278,279],{},"Your API key",[133,281,283],{"id":282},"launch","Launch",[10,285,286,287,290,291,78],{},"Click ",[60,288,289],{},"Review launch",", then ",[60,292,293],{},"Launch instance",[43,295,296],{"label":81},[83,297,299],{"className":85,"code":298,"language":87,"meta":88,"style":88},"featherless-cloud instance create \\\n  --name vllm \\\n  --template featherless\u002Frocm-vllm \\\n  --gpu-count 1 \\\n  --volume models:\u002Fworkspace \\\n  --port 8000 \\\n  --env HF_HOME=\u002Fworkspace\u002Fhf \\\n  --env VLLM_API_KEY=\"$VLLM_API_KEY\" \\\n  --command \"vllm serve Qwen\u002FQwen2.5-7B-Instruct --host 0.0.0.0 --port 8000\"\n",[14,300,301,314,325,336,347,358,369,380,397],{"__ignoreMap":88},[92,302,303,305,308,310],{"class":94,"line":95},[92,304,99],{"class":98},[92,306,307],{"class":102}," instance",[92,309,106],{"class":102},[92,311,313],{"class":312},"suJrU"," \\\n",[92,315,317,320,323],{"class":94,"line":316},2,[92,318,319],{"class":112},"  --name",[92,321,322],{"class":102}," vllm",[92,324,313],{"class":312},[92,326,328,331,334],{"class":94,"line":327},3,[92,329,330],{"class":112},"  --template",[92,332,333],{"class":102}," featherless\u002Frocm-vllm",[92,335,313],{"class":312},[92,337,339,342,345],{"class":94,"line":338},4,[92,340,341],{"class":112},"  --gpu-count",[92,343,344],{"class":112}," 1",[92,346,313],{"class":312},[92,348,350,353,356],{"class":94,"line":349},5,[92,351,352],{"class":112},"  --volume",[92,354,355],{"class":102}," models:\u002Fworkspace",[92,357,313],{"class":312},[92,359,361,364,367],{"class":94,"line":360},6,[92,362,363],{"class":112},"  --port",[92,365,366],{"class":112}," 8000",[92,368,313],{"class":312},[92,370,372,375,378],{"class":94,"line":371},7,[92,373,374],{"class":112},"  --env",[92,376,377],{"class":102}," HF_HOME=\u002Fworkspace\u002Fhf",[92,379,313],{"class":312},[92,381,383,385,388,392,395],{"class":94,"line":382},8,[92,384,374],{"class":112},[92,386,387],{"class":102}," VLLM_API_KEY=\"",[92,389,391],{"class":390},"sZEs4","$VLLM_API_KEY",[92,393,394],{"class":102},"\"",[92,396,313],{"class":312},[92,398,400,403],{"class":94,"line":399},9,[92,401,402],{"class":112},"  --command",[92,404,405],{"class":102}," \"vllm serve Qwen\u002FQwen2.5-7B-Instruct --host 0.0.0.0 --port 8000\"\n",[10,407,408,410,411,413],{},[14,409,264],{}," puts the downloaded model on the volume. ",[14,412,276],{}," makes vLLM reject requests without the key, because anyone can reach the public address.",[19,415,417],{"id":416},"step-3-wait-for-the-model-to-load","Step 3: Wait for the model to load",[10,419,420,421,424],{},"The first start downloads the image and the model, which takes several minutes. Open the instance's ",[60,422,423],{},"Logs"," tab, or follow them from the CLI, until vLLM reports that it's ready:",[83,426,428],{"className":85,"code":427,"language":87,"meta":88,"style":88},"featherless-cloud instance logs vllm --follow\n",[14,429,430],{"__ignoreMap":88},[92,431,432,434,436,439,441],{"class":94,"line":95},[92,433,99],{"class":98},[92,435,307],{"class":102},[92,437,438],{"class":102}," logs",[92,440,322],{"class":102},[92,442,443],{"class":112}," --follow\n",[83,445,448],{"className":446,"code":447,"language":229,"meta":88},[227],"(APIServer pid=44) INFO:     Application startup complete.\n",[14,449,447],{"__ignoreMap":88},[19,451,453],{"id":452},"step-4-call-the-model","Step 4: Call the model",[10,455,456,457,459,460,463,464,467],{},"Copy the address for port ",[14,458,213],{}," from the instance's ",[60,461,462],{},"Connect"," tab, or from ",[14,465,466],{},"featherless-cloud instance get vllm",". Then send a chat completion:",[83,469,471],{"className":85,"code":470,"language":87,"meta":88,"style":88},"curl https:\u002F\u002F\u003Caddress>\u002Fv1\u002Fchat\u002Fcompletions \\\n  -H \"Authorization: Bearer $VLLM_API_KEY\" \\\n  -H \"Content-Type: application\u002Fjson\" \\\n  -d '{\n    \"model\": \"Qwen\u002FQwen2.5-7B-Instruct\",\n    \"messages\": [{\"role\": \"user\", \"content\": \"Say hello in five words.\"}],\n    \"max_tokens\": 32\n  }'\n",[14,472,473,498,512,521,529,534,539,544],{"__ignoreMap":88},[92,474,475,478,481,484,487,490,493,496],{"class":94,"line":95},[92,476,477],{"class":98},"curl",[92,479,480],{"class":102}," https:\u002F\u002F",[92,482,483],{"class":312},"\u003C",[92,485,486],{"class":102},"addres",[92,488,489],{"class":390},"s",[92,491,492],{"class":312},">",[92,494,495],{"class":102},"\u002Fv1\u002Fchat\u002Fcompletions",[92,497,313],{"class":312},[92,499,500,503,506,508,510],{"class":94,"line":316},[92,501,502],{"class":112},"  -H",[92,504,505],{"class":102}," \"Authorization: Bearer ",[92,507,391],{"class":390},[92,509,394],{"class":102},[92,511,313],{"class":312},[92,513,514,516,519],{"class":94,"line":327},[92,515,502],{"class":112},[92,517,518],{"class":102}," \"Content-Type: application\u002Fjson\"",[92,520,313],{"class":312},[92,522,523,526],{"class":94,"line":338},[92,524,525],{"class":112},"  -d",[92,527,528],{"class":102}," '{\n",[92,530,531],{"class":94,"line":349},[92,532,533],{"class":102},"    \"model\": \"Qwen\u002FQwen2.5-7B-Instruct\",\n",[92,535,536],{"class":94,"line":360},[92,537,538],{"class":102},"    \"messages\": [{\"role\": \"user\", \"content\": \"Say hello in five words.\"}],\n",[92,540,541],{"class":94,"line":371},[92,542,543],{"class":102},"    \"max_tokens\": 32\n",[92,545,546],{"class":94,"line":382},[92,547,548],{"class":102},"  }'\n",[10,550,551],{},"The response follows the OpenAI format:",[83,553,557],{"className":554,"code":555,"language":556,"meta":88,"style":88},"language-json shiki shiki-themes github-dark-default","{\n  \"object\": \"chat.completion\",\n  \"model\": \"Qwen\u002FQwen2.5-7B-Instruct\",\n  \"choices\": [\n    {\n      \"index\": 0,\n      \"message\": { \"role\": \"assistant\", \"content\": \"Hello there, friend!\" },\n      \"finish_reason\": \"stop\"\n    }\n  ]\n}\n","json",[14,558,559,564,579,591,599,604,616,646,656,661,667],{"__ignoreMap":88},[92,560,561],{"class":94,"line":95},[92,562,563],{"class":390},"{\n",[92,565,566,570,573,576],{"class":94,"line":316},[92,567,569],{"class":568},"sPWt5","  \"object\"",[92,571,572],{"class":390},": ",[92,574,575],{"class":102},"\"chat.completion\"",[92,577,578],{"class":390},",\n",[92,580,581,584,586,589],{"class":94,"line":327},[92,582,583],{"class":568},"  \"model\"",[92,585,572],{"class":390},[92,587,588],{"class":102},"\"Qwen\u002FQwen2.5-7B-Instruct\"",[92,590,578],{"class":390},[92,592,593,596],{"class":94,"line":338},[92,594,595],{"class":568},"  \"choices\"",[92,597,598],{"class":390},": [\n",[92,600,601],{"class":94,"line":349},[92,602,603],{"class":390},"    {\n",[92,605,606,609,611,614],{"class":94,"line":360},[92,607,608],{"class":568},"      \"index\"",[92,610,572],{"class":390},[92,612,613],{"class":112},"0",[92,615,578],{"class":390},[92,617,618,621,624,627,629,632,635,638,640,643],{"class":94,"line":371},[92,619,620],{"class":568},"      \"message\"",[92,622,623],{"class":390},": { ",[92,625,626],{"class":568},"\"role\"",[92,628,572],{"class":390},[92,630,631],{"class":102},"\"assistant\"",[92,633,634],{"class":390},", ",[92,636,637],{"class":568},"\"content\"",[92,639,572],{"class":390},[92,641,642],{"class":102},"\"Hello there, friend!\"",[92,644,645],{"class":390}," },\n",[92,647,648,651,653],{"class":94,"line":382},[92,649,650],{"class":568},"      \"finish_reason\"",[92,652,572],{"class":390},[92,654,655],{"class":102},"\"stop\"\n",[92,657,658],{"class":94,"line":399},[92,659,660],{"class":390},"    }\n",[92,662,664],{"class":94,"line":663},10,[92,665,666],{"class":390},"  ]\n",[92,668,670],{"class":94,"line":669},11,[92,671,672],{"class":390},"}\n",[10,674,675,676,679,680,78],{},"Any OpenAI-compatible client works: set its base URL to ",[14,677,678],{},"https:\u002F\u002F\u003Caddress>\u002Fv1"," and its API key to your ",[14,681,276],{},[19,683,685],{"id":684},"clean-up","Clean up",[10,687,688],{},"Stop the instance when you don't need the server; the model stays cached on the volume for the next start. Delete both when you're done:",[83,690,692],{"className":85,"code":691,"language":87,"meta":88,"style":88},"featherless-cloud instance delete vllm\nfeatherless-cloud volume delete models\n",[14,693,694,706],{"__ignoreMap":88},[92,695,696,698,700,703],{"class":94,"line":95},[92,697,99],{"class":98},[92,699,307],{"class":102},[92,701,702],{"class":102}," delete",[92,704,705],{"class":102}," vllm\n",[92,707,708,710,712,714],{"class":94,"line":316},[92,709,99],{"class":98},[92,711,103],{"class":102},[92,713,702],{"class":102},[92,715,716],{"class":102}," models\n",[19,718,720],{"id":719},"next-steps","Next steps",[722,723,724,733],"card-group",{},[725,726,730],"card",{"icon":727,"title":728,"to":729},"i-lucide-globe","Expose ports","\u002Finstances\u002Fports",[10,731,732],{},"Limits and security for public addresses.",[725,734,738],{"icon":735,"title":736,"to":737},"i-lucide-box","Save it as a template","\u002Ftemplates\u002Fmanage",[10,739,740],{},"Deploy the same server again in one step.",[742,743,744],"style",{},"html pre.shiki code .sQhOw, html code.shiki .sQhOw{--shiki-default:#FFA657}html pre.shiki code .s9uIt, html code.shiki .s9uIt{--shiki-default:#A5D6FF}html pre.shiki code .sFSAA, html code.shiki .sFSAA{--shiki-default:#79C0FF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html pre.shiki code .suJrU, html code.shiki .suJrU{--shiki-default:#FF7B72}html pre.shiki code .sZEs4, html code.shiki .sZEs4{--shiki-default:#E6EDF3}html pre.shiki code .sPWt5, html code.shiki .sPWt5{--shiki-default:#7EE787}",{"title":88,"searchDepth":316,"depth":316,"links":746},[747,748,749,750,751,752,753],{"id":21,"depth":316,"text":22},{"id":36,"depth":316,"text":37},{"id":122,"depth":316,"text":123},{"id":416,"depth":316,"text":417},{"id":452,"depth":316,"text":453},{"id":684,"depth":316,"text":685},{"id":719,"depth":316,"text":720},"Serve an open model behind an OpenAI-compatible API on your own GPU.","md",{},true,"\u002Fexamples\u002Fvllm-server",{"title":5,"description":754},"examples\u002Fvllm-server","3V7YHNWHdS3EOTNTqV8LnXpP1rDLpqw-3trYFQ4cGK8",[763,767],{"title":764,"path":765,"description":766},"Develop in JupyterLab","\u002Fexamples\u002Fjupyterlab","Develop in notebooks on a dedicated GPU, with work that survives restarts.",{"title":768,"path":769,"description":770},"Fine-tune with a Run Job","\u002Fexamples\u002Ffine-tune-run-job","Train on a GPU you pay for only while training runs.",1794932598323]