[{"data":1,"prerenderedAt":461},["ShallowReactive",2],{"page-\u002Frun-jobs\u002Ftroubleshooting":3,"surround-\u002Frun-jobs\u002Ftroubleshooting":452},{"id":4,"title":5,"body":6,"description":444,"extension":445,"meta":446,"navigation":447,"path":448,"seo":449,"stem":450,"__hash__":451},"docs\u002Frun-jobs\u002Ftroubleshooting.md","Troubleshooting",{"type":7,"value":8,"toc":418},"minimark",[9,13,18,23,36,68,72,75,79,83,90,112,153,157,160,206,210,225,229,241,245,252,282,286,290,300,304,307,311,325,329,338,342,346,365,369,376,380,387,391,400,414],[10,11,12],"p",{},"This page lists common problems with Run Jobs, grouped by what you were doing, each with its cause and fix.",[14,15,17],"h2",{"id":16},"waiting-to-start","Waiting to start",[19,20,22],"h3",{"id":21},"a-run-job-waits-and-never-starts","A Run Job waits and never starts",[10,24,25,26,35],{},"Open it on the ",[27,28,34],"a",{"href":29,"rel":30,"target":33},"https:\u002F\u002Fconsole.cloud.featherless.ai\u002Frun-jobs",[31,32],"noopener","noreferrer","_blank","Run Jobs page"," to see what it's waiting for:",[37,38,39,47,58],"ul",{},[40,41,42,46],"li",{},[43,44,45],"strong",{},"Compute:"," no compute of the type it asked for is free. It starts when some frees up, or submit it with another GPU type.",[40,48,49,52,53,57],{},[43,50,51],{},"Your organization's limits:"," it starts when other instances or Run Jobs finish. See ",[27,54,56],{"href":55},"\u002Faccount\u002Forganizations#check-your-compute-limits","Check your compute limits",".",[40,59,60,63,64,57],{},[43,61,62],{},"Credit:"," your credit doesn't cover its time limit. Add credit, or submit it with a shorter ",[65,66,67],"code",{},"--timeout",[19,69,71],{"id":70},"a-run-job-takes-a-long-time-to-start-running","A Run Job takes a long time to start running",[10,73,74],{},"Before the command runs, the job downloads its image. The first run of a large image takes a few minutes; later runs are faster. Billing starts only when the container starts.",[14,76,78],{"id":77},"command-failures","Command failures",[19,80,82],{"id":81},"a-run-job-fails-right-away","A Run Job fails right away",[10,84,85,86,89],{},"Its status turns ",[43,87,88],{},"Failed"," within seconds of starting. Read its output and the message shown with the job. Common causes:",[37,91,92,98],{},[40,93,94,97],{},[43,95,96],{},"The command isn't in the image."," Check the image, or install what it needs as part of the command.",[40,99,100,103,104,107,108,111],{},[43,101,102],{},"The command uses shell syntax."," Commands run without a shell, so ",[65,105,106],{},"&&",", pipes and ",[65,109,110],{},"$VARIABLES"," aren't interpreted. Wrap them in a shell:",[113,114,119],"pre",{"className":115,"code":116,"language":117,"meta":118,"style":118},"language-bash shiki shiki-themes github-dark-default","featherless-cloud exec --gpu-count 1 -- bash -c \"pip install -r \u002Fworkspace\u002Frequirements.txt && python \u002Fworkspace\u002Ftrain.py\"\n","bash","",[65,120,121],{"__ignoreMap":118},[122,123,126,130,134,138,141,144,147,150],"span",{"class":124,"line":125},"line",1,[122,127,129],{"class":128},"sQhOw","featherless-cloud",[122,131,133],{"class":132},"s9uIt"," exec",[122,135,137],{"class":136},"sFSAA"," --gpu-count",[122,139,140],{"class":136}," 1",[122,142,143],{"class":136}," --",[122,145,146],{"class":132}," bash",[122,148,149],{"class":136}," -c",[122,151,152],{"class":132}," \"pip install -r \u002Fworkspace\u002Frequirements.txt && python \u002Fworkspace\u002Ftrain.py\"\n",[19,154,156],{"id":155},"the-job-cant-find-my-files","The job can't find my files",[10,158,159],{},"Your local files aren't uploaded with the job. Put them on a network volume and mount it, or build them into the image:",[113,161,163],{"className":115,"code":162,"language":117,"meta":118,"style":118},"featherless-cloud volume cp .\u002Ftrain.py vol:\u002F\u002Fmy-data\u002F\nfeatherless-cloud exec --gpu-count 1 --volume my-data:\u002Fworkspace -- python \u002Fworkspace\u002Ftrain.py\n",[65,164,165,181],{"__ignoreMap":118},[122,166,167,169,172,175,178],{"class":124,"line":125},[122,168,129],{"class":128},[122,170,171],{"class":132}," volume",[122,173,174],{"class":132}," cp",[122,176,177],{"class":132}," .\u002Ftrain.py",[122,179,180],{"class":132}," vol:\u002F\u002Fmy-data\u002F\n",[122,182,184,186,188,190,192,195,198,200,203],{"class":124,"line":183},2,[122,185,129],{"class":128},[122,187,133],{"class":132},[122,189,137],{"class":136},[122,191,140],{"class":136},[122,193,194],{"class":136}," --volume",[122,196,197],{"class":132}," my-data:\u002Fworkspace",[122,199,143],{"class":136},[122,201,202],{"class":132}," python",[122,204,205],{"class":132}," \u002Fworkspace\u002Ftrain.py\n",[19,207,209],{"id":208},"the-gpu-runs-out-of-memory","The GPU runs out of memory",[10,211,212,213,216,217,220,221,224],{},"Your framework reports that GPU memory is exhausted, for example ",[65,214,215],{},"HIP out of memory"," or ",[65,218,219],{},"CUDA out of memory"," in PyTorch. The model, its activations and the batch don't fit. Lower the batch size, load the model in a lower precision such as ",[65,222,223],{},"bfloat16",", or submit with more GPUs.",[19,226,228],{"id":227},"every-job-spends-minutes-installing-packages","Every job spends minutes installing packages",[10,230,231,232,236,237,57],{},"Each job starts from a clean container disk. Build the packages into a ",[27,233,235],{"href":234},"\u002Ftemplates\u002Fcustom-images","custom image"," and run the job from it, or save the setup as a ",[27,238,240],{"href":239},"\u002Ftemplates\u002Fmanage","template",[19,242,244],{"id":243},"my-secret-isnt-in-the-environment","My secret isn't in the environment",[10,246,247,248,251],{},"Secret variables are files under ",[65,249,250],{},"$FEATHERLESS_SECRETS_PATH",", not environment variables. Read them in your code, or export one in the command:",[113,253,255],{"className":115,"code":254,"language":117,"meta":118,"style":118},"featherless-cloud exec --gpu-count 1 --secret HF_TOKEN=hf_... -- bash -c 'export HF_TOKEN=\"$(cat \"$FEATHERLESS_SECRETS_PATH\u002FHF_TOKEN\")\" && python train.py'\n",[65,256,257],{"__ignoreMap":118},[122,258,259,261,263,265,267,270,273,275,277,279],{"class":124,"line":125},[122,260,129],{"class":128},[122,262,133],{"class":132},[122,264,137],{"class":136},[122,266,140],{"class":136},[122,268,269],{"class":136}," --secret",[122,271,272],{"class":132}," HF_TOKEN=hf_...",[122,274,143],{"class":136},[122,276,146],{"class":132},[122,278,149],{"class":136},[122,280,281],{"class":132}," 'export HF_TOKEN=\"$(cat \"$FEATHERLESS_SECRETS_PATH\u002FHF_TOKEN\")\" && python train.py'\n",[14,283,285],{"id":284},"time-limits-and-interruptions","Time limits and interruptions",[19,287,289],{"id":288},"a-run-job-timed-out","A Run Job timed out",[10,291,292,293,296,297,299],{},"Its status is ",[43,294,295],{},"Timed out",": it reached its time limit. Raise ",[65,298,67],{},", up to 24 hours, or save checkpoints to a network volume and continue in a new job.",[19,301,303],{"id":302},"the-work-needs-more-than-24-hours","The work needs more than 24 hours",[10,305,306],{},"A Run Job runs for at most 24 hours. Save checkpoints to a network volume, and submit a new job that resumes from the latest one when the previous one ends.",[19,308,310],{"id":309},"a-run-job-was-interrupted","A Run Job was interrupted",[10,312,292,313,316,317,320,321,57],{},[43,314,315],{},"Interrupted",": it was stopped to make room for other work. Submit with ",[65,318,319],{},"--preemption-retries \u003Cn>"," so it starts again, and write checkpoints to a network volume so a retry can resume instead of starting over. See ",[27,322,324],{"href":323},"\u002Frun-jobs#when-a-run-job-is-interrupted","When a Run Job is interrupted",[19,326,328],{"id":327},"a-run-job-stopped-when-credit-ran-out","A Run Job stopped when credit ran out",[10,330,331,332,337],{},"Running Run Jobs stop and fail when your balance reaches zero, and waiting ones keep waiting. Add credit on the ",[27,333,336],{"href":334,"rel":335,"target":33},"https:\u002F\u002Fconsole.cloud.featherless.ai\u002Fbilling",[31,32],"Billing page"," and submit the job again.",[14,339,341],{"id":340},"following-and-output","Following and output",[19,343,345],{"id":344},"i-pressed-ctrl-c-and-the-run-job-is-still-running","I pressed Ctrl-C and the Run Job is still running",[10,347,348,349,352,353,356,357,360,361,364],{},"Ctrl-C detaches ",[65,350,351],{},"exec"," from the job; it doesn't cancel it. Follow it again with ",[65,354,355],{},"featherless-cloud runjob logs \u003Cjob-id> --follow",", cancel it with ",[65,358,359],{},"featherless-cloud runjob cancel \u003Cjob-id>",", or submit with ",[65,362,363],{},"--cancel-on-interrupt"," so Ctrl-C cancels the job.",[19,366,368],{"id":367},"part-of-the-output-is-missing","Part of the output is missing",[10,370,371,372,375],{},"For each attempt, the first and last 50 MB of output are kept, and output is kept for 30 days. A job that started again after an interruption has one output per attempt; read an earlier one with ",[65,373,374],{},"featherless-cloud runjob logs \u003Cjob-id> --attempt 1",". Write anything you need to keep to a network volume.",[19,377,379],{"id":378},"a-run-jobs-files-are-gone","A Run Job's files are gone",[10,381,382,383,386],{},"Its container disk is erased when it ends. Mount a network volume with ",[65,384,385],{},"--volume \u003Cname>:\u003Cpath>"," and write results there.",[14,388,390],{"id":389},"getting-help","Getting help",[10,392,393,394,399],{},"If you're still stuck, ask us on ",[27,395,398],{"href":396,"rel":397,"target":33},"https:\u002F\u002Fdiscord.com\u002Finvite\u002F7gybCMPjVA",[31,32],"Discord",". Include:",[37,401,402,405,408],{},[40,403,404],{},"the Run Job's ID;",[40,406,407],{},"the command you submitted, without any secret values;",[40,409,410,411,57],{},"its status and the end of its output, from ",[65,412,413],{},"featherless-cloud runjob logs \u003Cjob-id>",[415,416,417],"style",{},"html pre.shiki code .sQhOw, html code.shiki .sQhOw{--shiki-default:#FFA657}html pre.shiki code .s9uIt, html code.shiki .s9uIt{--shiki-default:#A5D6FF}html pre.shiki code .sFSAA, html code.shiki .sFSAA{--shiki-default:#79C0FF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}",{"title":118,"searchDepth":183,"depth":183,"links":419},[420,425,432,438,443],{"id":16,"depth":183,"text":17,"children":421},[422,424],{"id":21,"depth":423,"text":22},3,{"id":70,"depth":423,"text":71},{"id":77,"depth":183,"text":78,"children":426},[427,428,429,430,431],{"id":81,"depth":423,"text":82},{"id":155,"depth":423,"text":156},{"id":208,"depth":423,"text":209},{"id":227,"depth":423,"text":228},{"id":243,"depth":423,"text":244},{"id":284,"depth":183,"text":285,"children":433},[434,435,436,437],{"id":288,"depth":423,"text":289},{"id":302,"depth":423,"text":303},{"id":309,"depth":423,"text":310},{"id":327,"depth":423,"text":328},{"id":340,"depth":183,"text":341,"children":439},[440,441,442],{"id":344,"depth":423,"text":345},{"id":367,"depth":423,"text":368},{"id":378,"depth":423,"text":379},{"id":389,"depth":183,"text":390},"Fix common Run Job problems, from jobs that never start to failed commands, time limits and missing output.","md",{},true,"\u002Frun-jobs\u002Ftroubleshooting",{"title":5,"description":444},"run-jobs\u002Ftroubleshooting","exUTcMImvl_b7Voco_1y71NdUTrVEWWnNzM-KIvdveI",[453,457],{"title":454,"path":455,"description":456},"Manage Run Jobs","\u002Frun-jobs\u002Fmanage","Track your Run Jobs, read their output, and stop the ones you don't need.",{"title":458,"path":459,"description":460},"Storage overview","\u002Fstorage","Keep datasets, models and checkpoints safe across instances and Run Jobs.",1794932598307]