Improve orchestrator initialization error handling and diagnostics

- Add comprehensive GatedRepoError handling during initialization
- Implement automatic fallback to API-only mode when local model loading fails
- Add detailed error logging with step-by-step diagnostics
- Improve 503 error responses with helpful messages
- Add per-component initialization logging for better debugging
- Handle gated repository errors gracefully with fallback to API mode

flask_api_standalone.py

@@ -164,36 +164,122 @@ def initialize_orchestrator():
         hf_token = os.getenv('HF_TOKEN', '')
         if not hf_token:
             logger.warning("HF_TOKEN not set - API fallback will be used if local models fail")
+        else:
+            logger.info(f"HF_TOKEN available (length: {len(hf_token)})")
+        
+        # Import GatedRepoError for better error handling
+        try:
+            from huggingface_hub.exceptions import GatedRepoError
+        except ImportError:
+            GatedRepoError = Exception
         
         # Initialize LLM Router with local model loading enabled
         logger.info("Initializing LLM Router with local GPU model loading...")
-        llm_router = LLMRouter(hf_token, use_local_models=True)
+        try:
+            llm_router = LLMRouter(hf_token, use_local_models=True)
+            logger.info("✓ LLM Router initialized")
+        except GatedRepoError as e:
+            logger.error(f"❌ Gated Repository Error during router initialization: {e}")
+            logger.error("   Falling back to API-only mode (local models disabled)")
+            # Try again without local models
+            llm_router = LLMRouter(hf_token, use_local_models=False)
+            logger.warning("⚠️  LLM Router initialized in API-only mode")
+        except Exception as e:
+            logger.error(f"❌ Failed to initialize LLM Router: {e}", exc_info=True)
+            logger.error("   Falling back to API-only mode")
+            try:
+                llm_router = LLMRouter(hf_token, use_local_models=False)
+                logger.warning("⚠️  LLM Router initialized in API-only mode after error")
+            except Exception as fallback_error:
+                logger.error(f"❌ Failed to initialize LLM Router even in API mode: {fallback_error}", exc_info=True)
+                raise
         
         logger.info("Initializing Agents...")
-        agents = {
-            'intent_recognition': create_intent_agent(llm_router),
-            'response_synthesis': create_synthesis_agent(llm_router),
-            'safety_check': create_safety_agent(llm_router),
-            'skills_identification': create_skills_identification_agent(llm_router)
-        }
+        try:
+            agents = {
+                'intent_recognition': create_intent_agent(llm_router),
+                'response_synthesis': create_synthesis_agent(llm_router),
+                'safety_check': create_safety_agent(llm_router),
+                'skills_identification': create_skills_identification_agent(llm_router)
+            }
+            logger.info("✓ All agents initialized")
+        except Exception as e:
+            logger.error(f"❌ Failed to initialize agents: {e}", exc_info=True)
+            raise
         
         logger.info("Initializing Context Manager...")
-        context_manager = EfficientContextManager(llm_router=llm_router)
+        try:
+            context_manager = EfficientContextManager(llm_router=llm_router)
+            logger.info("✓ Context Manager initialized")
+        except Exception as e:
+            logger.error(f"❌ Failed to initialize Context Manager: {e}", exc_info=True)
+            raise
         
         logger.info("Initializing Orchestrator...")
-        orchestrator = MVPOrchestrator(llm_router, context_manager, agents)
+        try:
+            orchestrator = MVPOrchestrator(llm_router, context_manager, agents)
+            logger.info("✓ Orchestrator initialized")
+        except Exception as e:
+            logger.error(f"❌ Failed to initialize Orchestrator: {e}", exc_info=True)
+            raise
         
         orchestrator_available = True
         logger.info("=" * 60)
         logger.info("✓ AI ORCHESTRATOR READY")
-        logger.info("  - Local GPU models enabled")
+        logger.info("  - Local GPU models enabled" if llm_router.use_local_models else "  - API-only mode (local models disabled)")
         logger.info("  - MAX_WORKERS: 4")
         logger.info("=" * 60)
         
         return True
         
+    except GatedRepoError as e:
+        logger.error("=" * 60)
+        logger.error("❌ GATED REPOSITORY ERROR DURING INITIALIZATION")
+        logger.error("=" * 60)
+        logger.error(f"Error: {e}")
+        logger.error("")
+        logger.error("SOLUTION:")
+        logger.error("1. Visit the model repository on Hugging Face")
+        logger.error("2. Click 'Agree and access repository'")
+        logger.error("3. Wait for approval (usually instant)")
+        logger.error("4. Ensure HF_TOKEN is set with your access token")
+        logger.error("=" * 60)
+        logger.warning("⚠️  Attempting to initialize in API-only mode...")
+        try:
+            # Try to initialize without local models
+            hf_token = os.getenv('HF_TOKEN', '')
+            from src.llm_router import LLMRouter
+            from src.agents.intent_agent import create_intent_agent
+            from src.agents.synthesis_agent import create_synthesis_agent
+            from src.agents.safety_agent import create_safety_agent
+            from src.agents.skills_identification_agent import create_skills_identification_agent
+            from src.orchestrator_engine import MVPOrchestrator
+            from src.context_manager import EfficientContextManager
+            
+            llm_router = LLMRouter(hf_token, use_local_models=False)
+            agents = {
+                'intent_recognition': create_intent_agent(llm_router),
+                'response_synthesis': create_synthesis_agent(llm_router),
+                'safety_check': create_safety_agent(llm_router),
+                'skills_identification': create_skills_identification_agent(llm_router)
+            }
+            context_manager = EfficientContextManager(llm_router=llm_router)
+            orchestrator = MVPOrchestrator(llm_router, context_manager, agents)
+            orchestrator_available = True
+            logger.info("✓ Orchestrator initialized in API-only mode")
+            return True
+        except Exception as fallback_error:
+            logger.error(f"❌ Failed to initialize in API-only mode: {fallback_error}", exc_info=True)
+            orchestrator_available = False
+            return False
     except Exception as e:
-        logger.error(f"Failed to initialize: {e}", exc_info=True)
+        logger.error("=" * 60)
+        logger.error("❌ FAILED TO INITIALIZE ORCHESTRATOR")
+        logger.error("=" * 60)
+        logger.error(f"Error type: {type(e).__name__}")
+        logger.error(f"Error message: {str(e)}")
+        logger.error("=" * 60)
+        logger.error("Full traceback:", exc_info=True)
         orchestrator_available = False
         return False
 
@@ -296,10 +382,12 @@ def chat():
         logger.info(f"Message length: {len(message)} chars, preview: {message[:100]}...")
         
         if not orchestrator_available or orchestrator is None:
+            logger.warning("Chat request received but orchestrator not ready")
             return jsonify({
                 'success': False,
                 'error': 'Orchestrator not ready',
-                'message': 'AI system is initializing. Please try again in a moment.'
+                'message': 'AI system is initializing. Please try again in a moment.',
+                'help': 'If this persists, check logs for initialization errors or try POST /api/initialize'
             }), 503
         
         # Process with orchestrator (async method)