[{"data":1,"prerenderedAt":985},["ShallowReactive",2],{"page-\u002Fexamples\u002Ffine-tune-run-job":3,"surround-\u002Fexamples\u002Ffine-tune-run-job":976},{"id":4,"title":5,"body":6,"description":969,"extension":970,"meta":971,"navigation":196,"path":972,"seo":973,"stem":974,"__hash__":975},"docs\u002Fexamples\u002Ffine-tune-run-job.md","Fine-tune with a Run Job",{"type":7,"value":8,"toc":960},"minimark",[9,13,18,34,38,119,123,134,710,717,737,741,752,841,844,852,865,869,899,905,908,912,915,932,936,956],[10,11,12],"p",{},"This example fine-tunes a small language model with LoRA as a Run Job. The training script and the trained adapter live on a network volume.",[14,15,17],"h2",{"id":16},"what-you-need","What you need",[19,20,21,31],"ul",{},[22,23,24,25,30],"li",{},"The ",[26,27,29],"a",{"href":28},"\u002Fcli","CLI",", signed in.",[22,32,33],{},"Credit for about an hour of one GPU. The run itself takes a few minutes.",[14,35,37],{"id":36},"step-1-create-a-network-volume","Step 1: Create a network volume",[39,40,42,79],"tabs",{"sync":41},"interface",[43,44,46],"tabs-item",{"label":45},"Console",[10,47,48,49,57,58,62,63,67,68,71,72,75,76,78],{},"Open the ",[26,50,56],{"href":51,"rel":52,"target":55},"https:\u002F\u002Fconsole.cloud.featherless.ai\u002Fstorage",[53,54],"noopener","noreferrer","_blank","Storage page",", click ",[59,60,61],"strong",{},"Create volume",", name it ",[64,65,66],"code",{},"training",", set ",[59,69,70],{},"Capacity"," to ",[64,73,74],{},"20",", and click ",[59,77,61],{},". It's ready for files after about a minute.",[43,80,81],{"label":29},[82,83,88],"pre",{"className":84,"code":85,"language":86,"meta":87,"style":87},"language-bash shiki shiki-themes github-dark-default","featherless-cloud volume create training --gb 20 --wait\n","bash","",[64,89,90],{"__ignoreMap":87},[91,92,95,99,103,106,109,113,116],"span",{"class":93,"line":94},"line",1,[91,96,98],{"class":97},"sQhOw","featherless-cloud",[91,100,102],{"class":101},"s9uIt"," volume",[91,104,105],{"class":101}," create",[91,107,108],{"class":101}," training",[91,110,112],{"class":111},"sFSAA"," --gb",[91,114,115],{"class":111}," 20",[91,117,118],{"class":111}," --wait\n",[14,120,122],{"id":121},"step-2-upload-the-training-script","Step 2: Upload the training script",[10,124,125,126,129,130,133],{},"Save this as ",[64,127,128],{},"finetune.py",". It trains a LoRA adapter for ",[64,131,132],{},"Qwen\u002FQwen2.5-0.5B"," on 1,000 instruction examples and saves it to the volume.",[82,135,139],{"className":136,"code":137,"filename":128,"language":138,"meta":87,"style":87},"language-python shiki shiki-themes github-dark-default","import torch\nfrom datasets import load_dataset\nfrom peft import LoraConfig, get_peft_model\nfrom transformers import AutoModelForCausalLM, AutoTokenizer, DataCollatorForLanguageModeling, Trainer, TrainingArguments\n\nmodel_id = \"Qwen\u002FQwen2.5-0.5B\"\noutput_dir = \"\u002Fworkspace\u002Foutputs\u002Fqwen-lora\"\n\ntokenizer = AutoTokenizer.from_pretrained(model_id)\nmodel = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=torch.bfloat16, device_map=\"cuda\")\nmodel = get_peft_model(model, LoraConfig(r=16, lora_alpha=32, target_modules=[\"q_proj\", \"v_proj\"], task_type=\"CAUSAL_LM\"))\nmodel.print_trainable_parameters()\n\ndataset = load_dataset(\"yahma\u002Falpaca-cleaned\", split=\"train[:1000]\")\n\ndef tokenize(example):\n    text = f\"### Instruction:\\n{example['instruction']}\\n\\n### Response:\\n{example['output']}{tokenizer.eos_token}\"\n    return tokenizer(text, truncation=True, max_length=512)\n\ndataset = dataset.map(tokenize, remove_columns=dataset.column_names)\n\ntrainer = Trainer(\n    model=model,\n    args=TrainingArguments(\n        output_dir=output_dir,\n        per_device_train_batch_size=8,\n        num_train_epochs=1,\n        learning_rate=2e-4,\n        bf16=True,\n        logging_steps=25,\n        save_strategy=\"no\",\n        report_to=[],\n    ),\n    train_dataset=dataset,\n    data_collator=DataCollatorForLanguageModeling(tokenizer, mlm=False),\n)\ntrainer.train()\nmodel.save_pretrained(output_dir)\nprint(f\"Saved the adapter to {output_dir}\")\n","python",[64,140,141,151,165,178,191,198,210,221,226,237,267,328,334,339,365,370,383,436,465,470,488,493,504,515,526,537,551,564,577,589,602,615,626,632,643,665,670,676,682],{"__ignoreMap":87},[91,142,143,147],{"class":93,"line":94},[91,144,146],{"class":145},"suJrU","import",[91,148,150],{"class":149},"sZEs4"," torch\n",[91,152,154,157,160,162],{"class":93,"line":153},2,[91,155,156],{"class":145},"from",[91,158,159],{"class":149}," datasets ",[91,161,146],{"class":145},[91,163,164],{"class":149}," load_dataset\n",[91,166,168,170,173,175],{"class":93,"line":167},3,[91,169,156],{"class":145},[91,171,172],{"class":149}," peft ",[91,174,146],{"class":145},[91,176,177],{"class":149}," LoraConfig, get_peft_model\n",[91,179,181,183,186,188],{"class":93,"line":180},4,[91,182,156],{"class":145},[91,184,185],{"class":149}," transformers ",[91,187,146],{"class":145},[91,189,190],{"class":149}," AutoModelForCausalLM, AutoTokenizer, DataCollatorForLanguageModeling, Trainer, TrainingArguments\n",[91,192,194],{"class":93,"line":193},5,[91,195,197],{"emptyLinePlaceholder":196},true,"\n",[91,199,201,204,207],{"class":93,"line":200},6,[91,202,203],{"class":149},"model_id ",[91,205,206],{"class":145},"=",[91,208,209],{"class":101}," \"Qwen\u002FQwen2.5-0.5B\"\n",[91,211,213,216,218],{"class":93,"line":212},7,[91,214,215],{"class":149},"output_dir ",[91,217,206],{"class":145},[91,219,220],{"class":101}," \"\u002Fworkspace\u002Foutputs\u002Fqwen-lora\"\n",[91,222,224],{"class":93,"line":223},8,[91,225,197],{"emptyLinePlaceholder":196},[91,227,229,232,234],{"class":93,"line":228},9,[91,230,231],{"class":149},"tokenizer ",[91,233,206],{"class":145},[91,235,236],{"class":149}," AutoTokenizer.from_pretrained(model_id)\n",[91,238,240,243,245,248,251,253,256,259,261,264],{"class":93,"line":239},10,[91,241,242],{"class":149},"model ",[91,244,206],{"class":145},[91,246,247],{"class":149}," AutoModelForCausalLM.from_pretrained(model_id, ",[91,249,250],{"class":97},"torch_dtype",[91,252,206],{"class":145},[91,254,255],{"class":149},"torch.bfloat16, ",[91,257,258],{"class":97},"device_map",[91,260,206],{"class":145},[91,262,263],{"class":101},"\"cuda\"",[91,265,266],{"class":149},")\n",[91,268,270,272,274,277,280,282,285,288,291,293,296,298,301,303,306,309,311,314,317,320,322,325],{"class":93,"line":269},11,[91,271,242],{"class":149},[91,273,206],{"class":145},[91,275,276],{"class":149}," get_peft_model(model, LoraConfig(",[91,278,279],{"class":97},"r",[91,281,206],{"class":145},[91,283,284],{"class":111},"16",[91,286,287],{"class":149},", ",[91,289,290],{"class":97},"lora_alpha",[91,292,206],{"class":145},[91,294,295],{"class":111},"32",[91,297,287],{"class":149},[91,299,300],{"class":97},"target_modules",[91,302,206],{"class":145},[91,304,305],{"class":149},"[",[91,307,308],{"class":101},"\"q_proj\"",[91,310,287],{"class":149},[91,312,313],{"class":101},"\"v_proj\"",[91,315,316],{"class":149},"], ",[91,318,319],{"class":97},"task_type",[91,321,206],{"class":145},[91,323,324],{"class":101},"\"CAUSAL_LM\"",[91,326,327],{"class":149},"))\n",[91,329,331],{"class":93,"line":330},12,[91,332,333],{"class":149},"model.print_trainable_parameters()\n",[91,335,337],{"class":93,"line":336},13,[91,338,197],{"emptyLinePlaceholder":196},[91,340,342,345,347,350,353,355,358,360,363],{"class":93,"line":341},14,[91,343,344],{"class":149},"dataset ",[91,346,206],{"class":145},[91,348,349],{"class":149}," load_dataset(",[91,351,352],{"class":101},"\"yahma\u002Falpaca-cleaned\"",[91,354,287],{"class":149},[91,356,357],{"class":97},"split",[91,359,206],{"class":145},[91,361,362],{"class":101},"\"train[:1000]\"",[91,364,266],{"class":149},[91,366,368],{"class":93,"line":367},15,[91,369,197],{"emptyLinePlaceholder":196},[91,371,373,376,380],{"class":93,"line":372},16,[91,374,375],{"class":145},"def",[91,377,379],{"class":378},"sc3cj"," tokenize",[91,381,382],{"class":149},"(example):\n",[91,384,386,389,391,394,397,400,403,406,409,412,415,417,419,422,424,427,430,433],{"class":93,"line":385},17,[91,387,388],{"class":149},"    text ",[91,390,206],{"class":145},[91,392,393],{"class":145}," f",[91,395,396],{"class":101},"\"### Instruction:",[91,398,399],{"class":145},"\\n{",[91,401,402],{"class":149},"example[",[91,404,405],{"class":101},"'instruction'",[91,407,408],{"class":149},"]",[91,410,411],{"class":145},"}\\n\\n",[91,413,414],{"class":101},"### Response:",[91,416,399],{"class":145},[91,418,402],{"class":149},[91,420,421],{"class":101},"'output'",[91,423,408],{"class":149},[91,425,426],{"class":145},"}{",[91,428,429],{"class":149},"tokenizer.eos_token",[91,431,432],{"class":145},"}",[91,434,435],{"class":101},"\"\n",[91,437,439,442,445,448,450,453,455,458,460,463],{"class":93,"line":438},18,[91,440,441],{"class":145},"    return",[91,443,444],{"class":149}," tokenizer(text, ",[91,446,447],{"class":97},"truncation",[91,449,206],{"class":145},[91,451,452],{"class":111},"True",[91,454,287],{"class":149},[91,456,457],{"class":97},"max_length",[91,459,206],{"class":145},[91,461,462],{"class":111},"512",[91,464,266],{"class":149},[91,466,468],{"class":93,"line":467},19,[91,469,197],{"emptyLinePlaceholder":196},[91,471,473,475,477,480,483,485],{"class":93,"line":472},20,[91,474,344],{"class":149},[91,476,206],{"class":145},[91,478,479],{"class":149}," dataset.map(tokenize, ",[91,481,482],{"class":97},"remove_columns",[91,484,206],{"class":145},[91,486,487],{"class":149},"dataset.column_names)\n",[91,489,491],{"class":93,"line":490},21,[91,492,197],{"emptyLinePlaceholder":196},[91,494,496,499,501],{"class":93,"line":495},22,[91,497,498],{"class":149},"trainer ",[91,500,206],{"class":145},[91,502,503],{"class":149}," Trainer(\n",[91,505,507,510,512],{"class":93,"line":506},23,[91,508,509],{"class":97},"    model",[91,511,206],{"class":145},[91,513,514],{"class":149},"model,\n",[91,516,518,521,523],{"class":93,"line":517},24,[91,519,520],{"class":97},"    args",[91,522,206],{"class":145},[91,524,525],{"class":149},"TrainingArguments(\n",[91,527,529,532,534],{"class":93,"line":528},25,[91,530,531],{"class":97},"        output_dir",[91,533,206],{"class":145},[91,535,536],{"class":149},"output_dir,\n",[91,538,540,543,545,548],{"class":93,"line":539},26,[91,541,542],{"class":97},"        per_device_train_batch_size",[91,544,206],{"class":145},[91,546,547],{"class":111},"8",[91,549,550],{"class":149},",\n",[91,552,554,557,559,562],{"class":93,"line":553},27,[91,555,556],{"class":97},"        num_train_epochs",[91,558,206],{"class":145},[91,560,561],{"class":111},"1",[91,563,550],{"class":149},[91,565,567,570,572,575],{"class":93,"line":566},28,[91,568,569],{"class":97},"        learning_rate",[91,571,206],{"class":145},[91,573,574],{"class":111},"2e-4",[91,576,550],{"class":149},[91,578,580,583,585,587],{"class":93,"line":579},29,[91,581,582],{"class":97},"        bf16",[91,584,206],{"class":145},[91,586,452],{"class":111},[91,588,550],{"class":149},[91,590,592,595,597,600],{"class":93,"line":591},30,[91,593,594],{"class":97},"        logging_steps",[91,596,206],{"class":145},[91,598,599],{"class":111},"25",[91,601,550],{"class":149},[91,603,605,608,610,613],{"class":93,"line":604},31,[91,606,607],{"class":97},"        save_strategy",[91,609,206],{"class":145},[91,611,612],{"class":101},"\"no\"",[91,614,550],{"class":149},[91,616,618,621,623],{"class":93,"line":617},32,[91,619,620],{"class":97},"        report_to",[91,622,206],{"class":145},[91,624,625],{"class":149},"[],\n",[91,627,629],{"class":93,"line":628},33,[91,630,631],{"class":149},"    ),\n",[91,633,635,638,640],{"class":93,"line":634},34,[91,636,637],{"class":97},"    train_dataset",[91,639,206],{"class":145},[91,641,642],{"class":149},"dataset,\n",[91,644,646,649,651,654,657,659,662],{"class":93,"line":645},35,[91,647,648],{"class":97},"    data_collator",[91,650,206],{"class":145},[91,652,653],{"class":149},"DataCollatorForLanguageModeling(tokenizer, ",[91,655,656],{"class":97},"mlm",[91,658,206],{"class":145},[91,660,661],{"class":111},"False",[91,663,664],{"class":149},"),\n",[91,666,668],{"class":93,"line":667},36,[91,669,266],{"class":149},[91,671,673],{"class":93,"line":672},37,[91,674,675],{"class":149},"trainer.train()\n",[91,677,679],{"class":93,"line":678},38,[91,680,681],{"class":149},"model.save_pretrained(output_dir)\n",[91,683,685,688,691,694,697,700,703,705,708],{"class":93,"line":684},39,[91,686,687],{"class":111},"print",[91,689,690],{"class":149},"(",[91,692,693],{"class":145},"f",[91,695,696],{"class":101},"\"Saved the adapter to ",[91,698,699],{"class":145},"{",[91,701,702],{"class":149},"output_dir",[91,704,432],{"class":145},[91,706,707],{"class":101},"\"",[91,709,266],{"class":149},[10,711,712,713,716],{},"Upload it to the volume. In the console, open the volume and click ",[59,714,715],{},"Upload files",", or use the CLI:",[82,718,720],{"className":84,"code":719,"language":86,"meta":87,"style":87},"featherless-cloud volume cp finetune.py vol:\u002F\u002Ftraining\u002F\n",[64,721,722],{"__ignoreMap":87},[91,723,724,726,728,731,734],{"class":93,"line":94},[91,725,98],{"class":97},[91,727,102],{"class":101},[91,729,730],{"class":101}," cp",[91,732,733],{"class":101}," finetune.py",[91,735,736],{"class":101}," vol:\u002F\u002Ftraining\u002F\n",[14,738,740],{"id":739},"step-3-run-the-training-job","Step 3: Run the training job",[10,742,743,744,747,748,751],{},"Submit the script as a Run Job on one GPU, with the volume mounted at ",[64,745,746],{},"\u002Fworkspace",". The command installs the libraries the script needs, then runs it. ",[64,749,750],{},"HF_HOME"," keeps the downloaded model on the volume, so the next run reuses it.",[82,753,755],{"className":84,"code":754,"language":86,"meta":87,"style":87},"featherless-cloud exec \\\n  --name finetune \\\n  --template featherless\u002Frocm-pytorch \\\n  --gpu-count 1 \\\n  --volume training:\u002Fworkspace \\\n  --env HF_HOME=\u002Fworkspace\u002Fhf \\\n  --timeout 3600 \\\n  -- bash -c \"pip install -q transformers peft datasets accelerate && python \u002Fworkspace\u002Ffinetune.py\"\n",[64,756,757,767,777,787,797,807,817,827],{"__ignoreMap":87},[91,758,759,761,764],{"class":93,"line":94},[91,760,98],{"class":97},[91,762,763],{"class":101}," exec",[91,765,766],{"class":145}," \\\n",[91,768,769,772,775],{"class":93,"line":153},[91,770,771],{"class":111},"  --name",[91,773,774],{"class":101}," finetune",[91,776,766],{"class":145},[91,778,779,782,785],{"class":93,"line":167},[91,780,781],{"class":111},"  --template",[91,783,784],{"class":101}," featherless\u002Frocm-pytorch",[91,786,766],{"class":145},[91,788,789,792,795],{"class":93,"line":180},[91,790,791],{"class":111},"  --gpu-count",[91,793,794],{"class":111}," 1",[91,796,766],{"class":145},[91,798,799,802,805],{"class":93,"line":193},[91,800,801],{"class":111},"  --volume",[91,803,804],{"class":101}," training:\u002Fworkspace",[91,806,766],{"class":145},[91,808,809,812,815],{"class":93,"line":200},[91,810,811],{"class":111},"  --env",[91,813,814],{"class":101}," HF_HOME=\u002Fworkspace\u002Fhf",[91,816,766],{"class":145},[91,818,819,822,825],{"class":93,"line":212},[91,820,821],{"class":111},"  --timeout",[91,823,824],{"class":111}," 3600",[91,826,766],{"class":145},[91,828,829,832,835,838],{"class":93,"line":223},[91,830,831],{"class":111},"  --",[91,833,834],{"class":101}," bash",[91,836,837],{"class":111}," -c",[91,839,840],{"class":101}," \"pip install -q transformers peft datasets accelerate && python \u002Fworkspace\u002Ffinetune.py\"\n",[10,842,843],{},"The CLI streams the output while the job runs. It ends like this:",[82,845,850],{"className":846,"code":848,"language":849,"meta":87},[847],"language-text","trainable params: 1,081,344 || all params: 495,114,112 || trainable%: 0.2184\n{'loss': '1.492', 'grad_norm': '0.6243', 'learning_rate': '0.0001616', 'epoch': '0.2'}\n...\n{'train_runtime': '8.94', 'train_samples_per_second': '111.9', 'train_steps_per_second': '13.98', 'train_loss': '1.458', 'epoch': '1'}\nSaved the adapter to \u002Fworkspace\u002Foutputs\u002Fqwen-lora\nRun Job jbkbsmhf6s: succeeded\n","text",[64,851,848],{"__ignoreMap":87},[10,853,854,855,860,861,864],{},"The job also appears on the console's ",[26,856,859],{"href":857,"rel":858,"target":55},"https:\u002F\u002Fconsole.cloud.featherless.ai\u002Frun-jobs",[53,54],"Run Jobs page",", with its logs. If you close the terminal, the job keeps running; follow it again with ",[64,862,863],{},"featherless-cloud runjob logs \u003Cjob-id> --follow",".",[14,866,868],{"id":867},"step-4-download-the-adapter","Step 4: Download the adapter",[82,870,872],{"className":84,"code":871,"language":86,"meta":87,"style":87},"featherless-cloud volume cp -r vol:\u002F\u002Ftraining\u002Foutputs\u002Fqwen-lora .\nls qwen-lora\n",[64,873,874,891],{"__ignoreMap":87},[91,875,876,878,880,882,885,888],{"class":93,"line":94},[91,877,98],{"class":97},[91,879,102],{"class":101},[91,881,730],{"class":101},[91,883,884],{"class":111}," -r",[91,886,887],{"class":101}," vol:\u002F\u002Ftraining\u002Foutputs\u002Fqwen-lora",[91,889,890],{"class":101}," .\n",[91,892,893,896],{"class":93,"line":153},[91,894,895],{"class":97},"ls",[91,897,898],{"class":101}," qwen-lora\n",[82,900,903],{"className":901,"code":902,"language":849,"meta":87},[847],"README.md  adapter_config.json  adapter_model.safetensors\n",[64,904,902],{"__ignoreMap":87},[10,906,907],{},"Load it with PEFT on top of the base model, or keep it on the volume and mount the volume into an instance that serves the model.",[14,909,911],{"id":910},"clean-up","Clean up",[10,913,914],{},"The Run Job's compute stopped when the script finished. The volume keeps billing until you delete it:",[82,916,918],{"className":84,"code":917,"language":86,"meta":87,"style":87},"featherless-cloud volume delete training\n",[64,919,920],{"__ignoreMap":87},[91,921,922,924,926,929],{"class":93,"line":94},[91,923,98],{"class":97},[91,925,102],{"class":101},[91,927,928],{"class":101}," delete",[91,930,931],{"class":101}," training\n",[14,933,935],{"id":934},"next-steps","Next steps",[937,938,939,948],"card-group",{},[940,941,945],"card",{"icon":942,"title":943,"to":944},"i-lucide-send","Submit a Run Job","\u002Frun-jobs\u002Fsubmit",[10,946,947],{},"Time limits, retries and detached runs.",[940,949,953],{"icon":950,"title":951,"to":952},"i-lucide-folder-up","Transfer files","\u002Fstorage\u002Ftransfer-files",[10,954,955],{},"Sync datasets and checkpoints with a volume.",[957,958,959],"style",{},"html pre.shiki code .sQhOw, html code.shiki .sQhOw{--shiki-default:#FFA657}html pre.shiki code .s9uIt, html code.shiki .s9uIt{--shiki-default:#A5D6FF}html pre.shiki code .sFSAA, html code.shiki .sFSAA{--shiki-default:#79C0FF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html pre.shiki code .suJrU, html code.shiki .suJrU{--shiki-default:#FF7B72}html pre.shiki code .sZEs4, html code.shiki .sZEs4{--shiki-default:#E6EDF3}html pre.shiki code .sc3cj, html code.shiki .sc3cj{--shiki-default:#D2A8FF}",{"title":87,"searchDepth":153,"depth":153,"links":961},[962,963,964,965,966,967,968],{"id":16,"depth":153,"text":17},{"id":36,"depth":153,"text":37},{"id":121,"depth":153,"text":122},{"id":739,"depth":153,"text":740},{"id":867,"depth":153,"text":868},{"id":910,"depth":153,"text":911},{"id":934,"depth":153,"text":935},"Train on a GPU you pay for only while training runs.","md",{},"\u002Fexamples\u002Ffine-tune-run-job",{"title":5,"description":969},"examples\u002Ffine-tune-run-job","d61MH74_Kmt6iKJu2twAEveo1Xv_jgENHHjxS7gTJvg",[977,981],{"title":978,"path":979,"description":980},"Serve an LLM with vLLM","\u002Fexamples\u002Fvllm-server","Serve an open model behind an OpenAI-compatible API on your own GPU.",{"title":982,"path":983,"description":984},"Launch GPUs from a CPU jumpbox","\u002Fexamples\u002Fcpu-jumpbox","Work from a small CPU instance that's always signed in, and start GPUs only when you need them.",1794932598324]