llama-3.3-70b

models/model_upload/llms/sglang-llama-3.3-70b-instruct-awq/1/model.py

@@ -107,6 +107,7 @@ def load_model(self):
 
     # if checkpoints section is in config.yaml file then checkpoints will be downloaded at this path during model upload time.
     # checkpoints = os.path.join(os.path.dirname(__file__), "checkpoints")
+
     checkpoints = "casperhansen/llama-3.3-70b-instruct-awq"
 
     try:

models/model_upload/llms/sglang-llama-3.3-70b-instruct-awq/config.yaml

@@ -2,21 +2,16 @@
 
 model:
   id: "sglang-llama-3_3-70b-instruct"
-  user_id: "user_id"
-  app_id: "app_id"
+  user_id: "meta"
+  app_id: "Llama-3"
   model_type_id: "text-to-text"
 
 build_info:
-  python_version: "3.10"
+  python_version: "3.11"
 
 inference_compute_info:
   cpu_limit: "2"
   cpu_memory: "16Gi"
   num_accelerators: 1
   accelerator_type: ["NVIDIA-L40S"]
-  accelerator_memory: "40Gi"
-
-# checkpoints:
-#   type: "huggingface"
-#   repo_id: "casperhansen/llama-3.3-70b-instruct-awq"
-#   hf_token: "token"
+  accelerator_memory: "42Gi"

models/model_upload/llms/sglang-llama-3.3-70b-instruct-awq/requirements.txt

@@ -1,15 +1,13 @@
 torch==2.5.1
-tokenizers==0.20.2
-transformers==4.46.2
-accelerate==0.34.2
+tokenizers>=0.21.0
+transformers>=4.47.0
+accelerate>=1.2.0
 scipy==1.10.1
-optimum==1.23.3
+optimum>=1.23.3
 xformers==0.0.28.post3
-einops==0.8.0
+protobuf==5.27.3
+einops>=0.8.0
 requests==2.32.2
-packaging
-ninja
-protobuf==3.20.0
 
 sglang[all]==0.3.6
 orjson==3.10.11