Trying to fix imports

app.py

@@ -29,6 +29,23 @@ os.environ["STREAMLIT_SERVER_HEADLESS"] = "true"
 os.environ["STREAMLIT_SERVER_ENABLE_CORS"] = "false" 
 os.environ["STREAMLIT_SERVER_ENABLE_XSRF_PROTECTION"] = "false"
 
+# Setup cache directories for model downloads (HuggingFace Spaces compatible)
+cache_base = "/tmp/.cache" if IS_HF_SPACES else str(Path.home() / ".cache")
+os.environ.setdefault("HF_HOME", f"{cache_base}/huggingface")
+os.environ.setdefault("TRANSFORMERS_CACHE", f"{cache_base}/huggingface/transformers")
+os.environ.setdefault("SENTENCE_TRANSFORMERS_HOME", f"{cache_base}/sentence-transformers")
+
+# Create cache directories
+for cache_dir in [
+    os.environ["HF_HOME"],
+    os.environ["TRANSFORMERS_CACHE"], 
+    os.environ["SENTENCE_TRANSFORMERS_HOME"]
+]:
+    try:
+        os.makedirs(cache_dir, exist_ok=True)
+    except Exception as e:
+        logger.warning(f"Could not create cache directory {cache_dir}: {e}")
+
 # HuggingFace Spaces environment detection
 IS_HF_SPACES = os.getenv("SPACE_ID") is not None
 IS_LOCAL_DEV = not IS_HF_SPACES
@@ -114,6 +131,42 @@ def setup_environment_display(capabilities):
     return capabilities
 
 
+def setup_models_if_needed():
+    """Setup models if needed for cloud deployment."""
+    try:
+        # Quick validation of critical dependencies
+        import rank_bm25
+        import pdfplumber
+        logger.info("✅ Critical dependencies available")
+        
+        # Check if we need to setup spaCy model
+        try:
+            import spacy
+            spacy.load("en_core_web_sm")
+            logger.info("✅ spaCy model available")
+        except OSError:
+            logger.info("📥 Setting up spaCy model...")
+            try:
+                import subprocess
+                result = subprocess.run([
+                    sys.executable, "-m", "spacy", "download", "en_core_web_sm"
+                ], capture_output=True, text=True, timeout=300)
+                if result.returncode == 0:
+                    logger.info("✅ spaCy model setup complete")
+                else:
+                    logger.warning("⚠️ spaCy model setup failed - entity extraction may be limited")
+            except Exception as e:
+                logger.warning(f"⚠️ spaCy model auto-setup failed: {e}")
+        except ImportError:
+            logger.warning("⚠️ spaCy not available")
+            
+    except ImportError as e:
+        logger.error(f"❌ Critical dependency missing: {e}")
+        st.error(f"Critical dependency missing: {e}")
+        st.info("Please install missing packages with: pip install -r requirements.txt")
+        st.stop()
+
+
 def main():
     """Main application entry point with Epic 2 enhanced capabilities."""
     
@@ -125,6 +178,9 @@ def main():
         initial_sidebar_state="expanded"
     )
     
+    # Setup models if needed
+    setup_models_if_needed()
+    
     # Check environment capabilities
     capabilities = check_environment_capabilities()
     setup_environment_display(capabilities)

requirements.txt

@@ -15,6 +15,7 @@ tokenizers>=0.13.0
 
 # Vector Search & Indexing
 faiss-cpu>=1.7.4
+weaviate-client>=3.15.0  # Alternative vector backend (optional but configured)
 
 # Document Processing (Essential only)
 PyMuPDF>=1.23.0
@@ -37,9 +38,12 @@ huggingface-hub>=0.16.0
 # Visualization (Lightweight)
 plotly>=5.17.0,<6.0.0
 
+# Core Dependencies (Previously Optional but Required)
+accelerate>=0.20.0  # HuggingFace model optimization (required for cloud deployment)
+pdfplumber>=0.10.0  # Advanced PDF parsing (required for document processing)
+rank-bm25>=0.2.2    # BM25 sparse retrieval (required for Epic 2 hybrid search)
+
 # Optional dependencies for local development only
-# accelerate>=0.20.0  # Uncomment for local GPU acceleration
-pdfplumber>=0.10.0  # Uncomment for advanced PDF parsing
 # fastapi>=0.104.0    # Uncomment for API server
 # uvicorn>=0.24.0     # Uncomment for API server
 # ollama>=0.5.0       # Uncomment for local LLM

scripts/setup_models.py

@@ -0,0 +1,206 @@
+#!/usr/bin/env python3
+"""
+Model Setup Script for Enhanced RAG Demo
+
+This script handles automatic downloading and setup of required models
+for deployment environments like HuggingFace Spaces where models may not
+be pre-installed.
+
+Usage:
+    python scripts/setup_models.py
+
+Environment Variables:
+    SKIP_MODEL_DOWNLOAD: Set to '1' to skip model downloads
+    SPACY_MODEL: Override default spaCy model (default: en_core_web_sm)
+"""
+
+import os
+import sys
+import logging
+import subprocess
+import time
+from pathlib import Path
+from typing import List, Dict, Any, Optional
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(levelname)s - %(message)s'
+)
+logger = logging.getLogger(__name__)
+
+def check_spacy_model(model_name: str = "en_core_web_sm") -> bool:
+    """
+    Check if spaCy model is available.
+    
+    Args:
+        model_name: Name of the spaCy model to check
+        
+    Returns:
+        True if model is available, False otherwise
+    """
+    try:
+        import spacy
+        spacy.load(model_name)
+        logger.info(f"✅ spaCy model '{model_name}' is available")
+        return True
+    except OSError:
+        logger.warning(f"❌ spaCy model '{model_name}' not found")
+        return False
+    except ImportError:
+        logger.warning("❌ spaCy not installed")
+        return False
+    except Exception as e:
+        logger.warning(f"❌ Error checking spaCy model: {e}")
+        return False
+
+def download_spacy_model(model_name: str = "en_core_web_sm", timeout: int = 300) -> bool:
+    """
+    Download spaCy model.
+    
+    Args:
+        model_name: Name of the spaCy model to download
+        timeout: Download timeout in seconds
+        
+    Returns:
+        True if download successful, False otherwise
+    """
+    try:
+        logger.info(f"📥 Downloading spaCy model '{model_name}'...")
+        
+        result = subprocess.run([
+            sys.executable, "-m", "spacy", "download", model_name
+        ], capture_output=True, text=True, timeout=timeout)
+        
+        if result.returncode == 0:
+            logger.info(f"✅ Successfully downloaded spaCy model '{model_name}'")
+            return True
+        else:
+            logger.error(f"❌ Failed to download spaCy model: {result.stderr}")
+            return False
+            
+    except subprocess.TimeoutExpired:
+        logger.error(f"❌ spaCy model download timed out after {timeout} seconds")
+        return False
+    except Exception as e:
+        logger.error(f"❌ Error downloading spaCy model: {e}")
+        return False
+
+def setup_cache_directories() -> None:
+    """
+    Set up cache directories for models with proper permissions.
+    """
+    cache_dirs = [
+        os.environ.get('TRANSFORMERS_CACHE', '/tmp/.cache/huggingface/transformers'),
+        os.environ.get('HF_HOME', '/tmp/.cache/huggingface'),
+        os.environ.get('SENTENCE_TRANSFORMERS_HOME', '/tmp/.cache/sentence-transformers'),
+    ]
+    
+    for cache_dir in cache_dirs:
+        try:
+            os.makedirs(cache_dir, exist_ok=True)
+            logger.info(f"📁 Created cache directory: {cache_dir}")
+        except Exception as e:
+            logger.warning(f"⚠️ Could not create cache directory {cache_dir}: {e}")
+
+def validate_python_packages() -> Dict[str, bool]:
+    """
+    Validate that required Python packages are installed.
+    
+    Returns:
+        Dictionary mapping package names to availability status
+    """
+    required_packages = {
+        'rank_bm25': 'rank_bm25',
+        'pdfplumber': 'pdfplumber', 
+        'sentence_transformers': 'sentence_transformers',
+        'transformers': 'transformers',
+        'spacy': 'spacy',
+        'huggingface_hub': 'huggingface_hub',
+        'faiss': 'faiss',
+        'accelerate': 'accelerate'  # Optional but recommended
+    }
+    
+    status = {}
+    
+    for display_name, import_name in required_packages.items():
+        try:
+            __import__(import_name)
+            status[display_name] = True
+            logger.info(f"✅ {display_name} is available")
+        except ImportError:
+            status[display_name] = False
+            logger.error(f"❌ {display_name} is not installed")
+    
+    return status
+
+def main() -> int:
+    """
+    Main setup function.
+    
+    Returns:
+        Exit code (0 for success, 1 for failure)
+    """
+    logger.info("🚀 Starting Enhanced RAG Demo model setup...")
+    
+    # Check if model download should be skipped
+    skip_download = os.environ.get('SKIP_MODEL_DOWNLOAD', '').lower() in ('1', 'true', 'yes')
+    if skip_download:
+        logger.info("⏭️ Skipping model downloads (SKIP_MODEL_DOWNLOAD set)")
+        return 0
+    
+    success = True
+    
+    # 1. Validate Python packages
+    logger.info("📦 Validating Python packages...")
+    package_status = validate_python_packages()
+    
+    critical_packages = ['rank_bm25', 'pdfplumber', 'sentence_transformers', 'transformers', 'spacy']
+    missing_critical = [pkg for pkg in critical_packages if not package_status.get(pkg, False)]
+    
+    if missing_critical:
+        logger.error(f"❌ Critical packages missing: {', '.join(missing_critical)}")
+        logger.error("Please install missing packages with: pip install -r requirements.txt")
+        success = False
+    
+    # 2. Setup cache directories
+    logger.info("📁 Setting up cache directories...")
+    setup_cache_directories()
+    
+    # 3. Handle spaCy model
+    spacy_model = os.environ.get('SPACY_MODEL', 'en_core_web_sm')
+    logger.info(f"🔤 Checking spaCy model: {spacy_model}")
+    
+    if package_status.get('spacy', False):
+        if not check_spacy_model(spacy_model):
+            logger.info(f"📥 Attempting to download spaCy model '{spacy_model}'...")
+            if not download_spacy_model(spacy_model):
+                logger.error(f"❌ Failed to download spaCy model '{spacy_model}'")
+                logger.warning("⚠️ Entity extraction features may be limited")
+                # Don't fail completely - this is non-critical for basic functionality
+    else:
+        logger.warning("⚠️ spaCy not available - entity extraction will be disabled")
+    
+    # 4. Test model loading (basic validation)
+    if package_status.get('sentence_transformers', False):
+        try:
+            logger.info("🧪 Testing sentence-transformers model loading...")
+            from sentence_transformers import SentenceTransformer
+            
+            # Try to load a small model for validation
+            model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2', cache_folder='/tmp/.cache/sentence-transformers')
+            logger.info("✅ sentence-transformers model loading successful")
+            del model  # Free memory
+        except Exception as e:
+            logger.warning(f"⚠️ sentence-transformers model loading failed: {e}")
+    
+    if success:
+        logger.info("🎉 Model setup completed successfully!")
+        return 0
+    else:
+        logger.error("💥 Model setup encountered errors")
+        return 1
+
+if __name__ == "__main__":
+    exit_code = main()
+    sys.exit(exit_code)

src/components/embedders/models/sentence_transformer_model.py

@@ -153,21 +153,51 @@ class SentenceTransformerModel(EmbeddingModel, ConfigurableEmbedderComponent):
                     trust_remote_code=self.trust_remote_code
                 )
             else:
-                # Use default cache behavior with fallback
-                try:
-                    model = SentenceTransformer(
-                        self.model_name,
-                        trust_remote_code=self.trust_remote_code
-                    )
-                except Exception as e:
-                    # Fallback to explicit cache directory
-                    cache_dir = os.environ.get('SENTENCE_TRANSFORMERS_HOME', '/tmp/.cache/sentence-transformers')
-                    os.makedirs(cache_dir, exist_ok=True)
-                    model = SentenceTransformer(
-                        self.model_name,
-                        cache_folder=cache_dir,
-                        trust_remote_code=self.trust_remote_code
-                    )
+                # Use default cache behavior with multiple fallbacks for cloud environments
+                cache_attempts = [
+                    # Try default cache first
+                    None,
+                    # HuggingFace Spaces compatible paths
+                    os.environ.get('SENTENCE_TRANSFORMERS_HOME', '/tmp/.cache/sentence-transformers'),
+                    '/tmp/.cache/sentence-transformers',
+                    '/app/.cache/sentence-transformers',  # Common in containerized environments
+                    './models/cache',  # Local fallback
+                    '/tmp/models'  # Final fallback
+                ]
+                
+                model = None
+                last_error = None
+                
+                for cache_dir in cache_attempts:
+                    try:
+                        if cache_dir:
+                            # Ensure cache directory exists and is writable
+                            os.makedirs(cache_dir, exist_ok=True)
+                            # Test if directory is writable
+                            test_file = os.path.join(cache_dir, '.write_test')
+                            with open(test_file, 'w') as f:
+                                f.write('test')
+                            os.remove(test_file)
+                            
+                            model = SentenceTransformer(
+                                self.model_name,
+                                cache_folder=cache_dir,
+                                trust_remote_code=self.trust_remote_code
+                            )
+                        else:
+                            model = SentenceTransformer(
+                                self.model_name,
+                                trust_remote_code=self.trust_remote_code
+                            )
+                        break  # Success - exit loop
+                        
+                    except (OSError, PermissionError, Exception) as e:
+                        last_error = e
+                        logger.warning(f"Cache attempt failed for {cache_dir}: {e}")
+                        continue
+                
+                if model is None:
+                    raise RuntimeError(f"Failed to load model with any cache configuration. Last error: {last_error}")
             
             # Move to device and set to eval mode
             model = model.to(self.device)

src/components/retrievers/graph/entity_extraction.py

@@ -115,10 +115,35 @@ class EntityExtractor:
             
         except OSError as e:
             if "Can't find model" in str(e):
-                raise EntityExtractionError(
-                    f"spaCy model '{self.config.model}' not found. "
-                    f"Install with: python -m spacy download {self.config.model}"
-                ) from e
+                logger.warning(f"spaCy model '{self.config.model}' not found. Attempting to download...")
+                try:
+                    # Attempt to download the model
+                    import subprocess
+                    import sys
+                    
+                    result = subprocess.run([
+                        sys.executable, "-m", "spacy", "download", self.config.model
+                    ], capture_output=True, text=True, timeout=300)  # 5 minute timeout
+                    
+                    if result.returncode == 0:
+                        logger.info(f"Successfully downloaded spaCy model '{self.config.model}'")
+                        # Try loading again
+                        self.nlp = spacy.load(self.config.model)
+                        logger.info(f"Loaded spaCy model: {self.config.model}")
+                    else:
+                        logger.error(f"Failed to download spaCy model: {result.stderr}")
+                        raise EntityExtractionError(
+                            f"spaCy model '{self.config.model}' not found and auto-download failed. "
+                            f"Manual install: python -m spacy download {self.config.model}"
+                        ) from e
+                        
+                except (subprocess.TimeoutExpired, subprocess.CalledProcessError, Exception) as download_error:
+                    logger.error(f"Model download failed: {download_error}")
+                    raise EntityExtractionError(
+                        f"spaCy model '{self.config.model}' not found and auto-download failed. "
+                        f"Manual install: python -m spacy download {self.config.model}. "
+                        f"Download error: {download_error}"
+                    ) from e
             else:
                 raise EntityExtractionError(f"Failed to load spaCy model: {str(e)}") from e
         except Exception as e:

src/core/component_factory.py

@@ -147,7 +147,36 @@ class ComponentFactory:
             return component_class
             
         except (ImportError, AttributeError) as e:
-            raise ImportError(f"Failed to import {module_path}: {e}") from e
+            # Enhanced error handling with dependency suggestions
+            error_msg = str(e)
+            suggestions = []
+            
+            # Check for specific missing dependencies and provide actionable suggestions
+            if "rank_bm25" in error_msg or "BM25" in error_msg:
+                suggestions.append("Install rank-bm25: pip install rank-bm25>=0.2.2")
+            elif "pdfplumber" in error_msg:
+                suggestions.append("Install pdfplumber: pip install pdfplumber>=0.10.0")
+            elif "sentence_transformers" in error_msg:
+                suggestions.append("Install sentence-transformers: pip install sentence-transformers>=2.2.0")
+            elif "transformers" in error_msg:
+                suggestions.append("Install transformers: pip install transformers>=4.30.0")
+            elif "spacy" in error_msg:
+                suggestions.append("Install spacy: pip install spacy>=3.7.0")
+                suggestions.append("Download spacy model: python -m spacy download en_core_web_sm")
+            elif "weaviate" in error_msg:
+                suggestions.append("Install weaviate-client: pip install weaviate-client>=3.15.0")
+            elif "huggingface_hub" in error_msg:
+                suggestions.append("Install huggingface-hub: pip install huggingface-hub>=0.16.0")
+            elif "accelerate" in error_msg:
+                suggestions.append("Install accelerate: pip install accelerate>=0.20.0")
+            
+            enhanced_msg = f"Failed to import {module_path}: {e}"
+            if suggestions:
+                enhanced_msg += f"\n\nSuggested fixes:\n" + "\n".join(f"  - {s}" for s in suggestions)
+                enhanced_msg += f"\n\nAlternatively, install all requirements: pip install -r requirements.txt"
+            
+            logger.error(enhanced_msg)
+            raise ImportError(enhanced_msg) from e
     
     @classmethod
     def get_performance_metrics(cls) -> Dict[str, Dict[str, Any]]: