fixes

app.py

@@ -16,12 +16,11 @@ llm = LlmBackend()
 _lock = threading.Lock()
     
 SYSTEM_PROMPT = os.environ.get('SYSTEM_PROMPT') or "Ты — русскоязычный автоматический ассистент. Ты максимально точно и отвечаешь на запросы пользователя, используя русский язык."
-CONTEXT_SIZE = os.environ.get('CONTEXT_SIZE') or 500
+CONTEXT_SIZE = int(os.environ.get('CONTEXT_SIZE', '500'))
 HF_CACHE_DIR = os.environ.get('HF_CACHE_DIR') or '/home/user/app/.cache'
-USE_SYSTEM_PROMPT = os.environ.get('USE_SYSTEM_PROMPT') or False
-ENABLE_GPU = os.environ.get('ENABLE_GPU') or False
-GPU_LAYERS = os.environ.get('GPU_LAYERS') or 0
-N_GQA = os.environ.get('N_GQA') or None #must be set to 8 for 70b models
+USE_SYSTEM_PROMPT = os.environ.get('USE_SYSTEM_PROMPT', '').lower() == "true" or False
+ENABLE_GPU = os.environ.get('ENABLE_GPU', '').lower() == "true" or False
+GPU_LAYERS = int(os.environ.get('GPU_LAYERS', '0'))
 CHAT_FORMAT = os.environ.get('CHAT_FORMAT') or 'llama-2'
 REPO_NAME = os.environ.get('REPO_NAME') or 'IlyaGusev/saiga2_7b_gguf'
 MODEL_NAME = os.environ.get('MODEL_NAME') or 'model-q4_K.gguf'
@@ -154,7 +153,7 @@ def generate_response():
     return Response(generate_and_log_tokens(user_request='1', generator=generator), content_type='text/plain', status=200, direct_passthrough=True)
 
 def init_model():
-    llm.load_model(model_path=MODEL_PATH, context_size=CONTEXT_SIZE, enable_gpu=ENABLE_GPU, gpu_layer_number=GPU_LAYERS, n_gqa=N_GQA)
+    llm.load_model(model_path=MODEL_PATH, context_size=CONTEXT_SIZE, enable_gpu=ENABLE_GPU, gpu_layer_number=GPU_LAYERS)
     
 # Function to check if no requests were made in the last 5 minutes
 def check_last_request_time():
@@ -171,9 +170,9 @@ if __name__ == "__main__":
     
     init_model()
     
-    scheduler = BackgroundScheduler()
-    scheduler.add_job(check_last_request_time, trigger='interval', minutes=1)
-    scheduler.start()
+    # scheduler = BackgroundScheduler()
+    # scheduler.add_job(check_last_request_time, trigger='interval', minutes=1)
+    # scheduler.start()
     
     app.run(host="0.0.0.0", port=7860, debug=True, threaded=True)
     

llm_backend.py

@@ -34,14 +34,13 @@ class LlmBackend:
     def is_model_loaded(self):
         return self._model is not None
     
-    def load_model(self, model_path, context_size=2000, enable_gpu=True, gpu_layer_number=35, n_gqa=8, chat_format='llama-2'):
+    def load_model(self, model_path, context_size=2000, enable_gpu=True, gpu_layer_number=35, chat_format='llama-2'):
         log.info('load_model - started')
         self._model_params = {}
         self._model_params['model_path'] = model_path
         self._model_params['context_size'] = context_size
         self._model_params['enable_gpu'] = enable_gpu
         self._model_params['gpu_layer_number'] = gpu_layer_number
-        self._model_params['n_gqa'] = n_gqa
         self._model_params['chat_format'] = chat_format
         
         if self._model is not None:
@@ -57,9 +56,8 @@ class LlmBackend:
                     #n_batch=100,
                     logits_all=True,
                     #n_threads=12,
-                    verbose=False,
-                    n_gpu_layers=gpu_layer_number,
-                    n_gqa=n_gqa       #must be set for 70b models
+                    verbose=True,
+                    n_gpu_layers=gpu_layer_number
                 )
                 log.info('load_model - finished')
                 return self._model
@@ -72,8 +70,7 @@ class LlmBackend:
                     #n_batch=100,
                     logits_all=True,
                     #n_threads=12,
-                    verbose=False,
-                    n_gqa=n_gqa       #must be set for 70b models
+                    verbose=True
                 )
                 log.info('load_model - finished')
                 return self._model