add deepspeed

runpod/runpod-worker-oobabooga-api/Dockerfile

@@ -60,6 +60,7 @@ RUN --mount=type=cache,target=/root/.cache,sharing=locked pip3 install --upgrade
     pip3 install safetensors && \
     pip3 install sentencepiece && \
     pip3 install diffusers && \
+    pip3 install deepspeed && \
     pip3 install accelerate xformers triton && \
     pip3 install git+https://github.com/huggingface/transformers.git && \
     pip3 install huggingface-hub && \
@@ -123,6 +124,7 @@ RUN --mount=type=cache,target=/root/.cache,sharing=locked pip3 install --upgrade
     pip3 install safetensors && \
     pip3 install sentencepiece && \
     pip3 install diffusers && \
+    pip3 install deepspeed && \
     pip3 install accelerate xformers triton && \
     pip3 install git+https://github.com/huggingface/transformers.git && \
     pip3 install rwkv && \

runpod/runpod-worker-oobabooga-api/start.sh

@@ -4,7 +4,8 @@ export PYTHONUNBUFFERED=1
 
 cd /workspace/text-generation-webui
 echo "starting api"
-python3 server.py --listen --no-stream --extensions api &
+python3 server.py --listen --no-stream --extensions api --auto-devices &
+# --deepspeed --gpu-memory 24
 cd /workspace
 echo "starting worker"
 python3 -u runpod_infer.py