embedding fix v2

Author: TalhaMudo

README.md

@@ -42,7 +42,8 @@ The dashboard allows:
 - pausing and resuming existing jobs,
 - changing **global queue limit** across all jobs,
 - changing **per-job request rate** live from each job card,
-- starting, pausing, and resuming **embedding jobs** with configurable embed speed and max-page limit,
+- starting and pausing a single **embedding engine** with configurable embed speed and max-page limit,
+- clearing all stored embeddings with confirmation when a full reset is needed,
 - inspecting detailed job state (frontier preview, counters, recent events),
 - viewing global metrics (processed/discovered/duplicates/failed, queue pressure, workers),
 - viewing embedding progress metrics (embedded/remaining/failed and `% embedded`).
@@ -57,7 +58,7 @@ The dashboard allows:
   - `job_visited` and `job_frontier` for resumable crawl state,
   - `job_events` for job timeline details,
   - `page_embeddings` for semantic vectors,
-  - `embedding_jobs` for resumable embedding job state.
+  - semantic engine runtime state in backend memory, with progress derived from DB counts.
 
 ## API Endpoints
 
@@ -96,23 +97,20 @@ The dashboard allows:
 
 ### Embeddings
 
-- `POST /embeddings/jobs/start`  
-  Starts a manual embedding job with configurable speed and max-page scope.
+- `GET /embeddings/status`  
+  Returns current embedding engine state and progress.
 
-- `GET /embeddings/jobs`  
-  Lists embedding jobs and progress.
+- `POST /embeddings/start`  
+  Starts (or continues) embedding missing pages with configurable speed and max-page scope.
 
-- `GET /embeddings/jobs/{job_id}`  
-  Returns full embedding job status and counters.
+- `POST /embeddings/pause`  
+  Pauses the current embedding engine run.
 
-- `POST /embeddings/jobs/{job_id}/pause`  
-  Pauses a running embedding job.
+- `POST /embeddings/rate-limit`  
+  Updates embedding speed (`pages/s`) for the engine.
 
-- `POST /embeddings/jobs/{job_id}/resume`  
-  Resumes a paused embedding job.
-
-- `POST /embeddings/jobs/{job_id}/rate-limit`  
-  Updates embedding speed (`pages/s`) for a job.
+- `POST /embeddings/clear`  
+  Deletes all stored embeddings so semantic indexing can be rebuilt from zero.
 
 ## Backpressure and Load Management
 

backend/app.py

@@ -211,6 +211,12 @@ async def update_embedding_engine_rate_limit(request: EmbeddingRateLimitRequest)
     return await _serialize_embedding_status()
 
 
+@app.post("/embeddings/clear", response_model=EmbeddingStatusResponse)
+async def clear_embeddings() -> EmbeddingStatusResponse:
+    await semantic_index_service.clear_embeddings()
+    return await _serialize_embedding_status()
+
+
 def _serialize_job(job_id: str) -> JobDetailResponse:
     ctx = crawler_service.get_job_context(job_id)
     if ctx is None:

backend/semantic_index.py

@@ -8,7 +8,14 @@
 from datetime import datetime
 from typing import Dict, List, Tuple
 
-from .storage import count_pages, count_page_embeddings, load_embedding_targets, load_page_embeddings, save_page_embedding
+from .storage import (
+    count_page_embeddings,
+    count_pages,
+    delete_page_embeddings,
+    load_embedding_targets,
+    load_page_embeddings,
+    save_page_embedding,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -94,6 +101,20 @@ async def update_rate_limit(self, rate_limit_per_sec: float) -> EmbeddingEngineS
         self._updated_at = datetime.utcnow()
         return await self.get_engine_status()
 
+    async def clear_embeddings(self) -> EmbeddingEngineStatus:
+        if self._task and not self._task.done():
+            self._task.cancel()
+            await asyncio.gather(self._task, return_exceptions=True)
+        delete_page_embeddings(DEFAULT_EMBEDDING_MODEL)
+        self._embeddings.clear()
+        self._meta.clear()
+        self._failed_pages = 0
+        self._error_message = None
+        self._status = "idle"
+        self._updated_at = datetime.utcnow()
+        self._task = None
+        return await self.get_engine_status()
+
     async def search(self, query: str, limit: int | None = None) -> List[Tuple[str, str, int, float, str]]:
         query = query.strip()
         if not query:

backend/storage.py

@@ -585,3 +585,18 @@ def count_page_embeddings(model_name: str) -> int:
     finally:
         conn.close()
 
+
+def delete_page_embeddings(model_name: str | None = None) -> int:
+    conn = _connect()
+    try:
+        cur = conn.cursor()
+        if model_name:
+            cur.execute("DELETE FROM page_embeddings WHERE model_name = ?", (model_name,))
+        else:
+            cur.execute("DELETE FROM page_embeddings")
+        deleted = int(cur.rowcount if cur.rowcount is not None else 0)
+        conn.commit()
+        return deleted
+    finally:
+        conn.close()
+

frontend/src/App.tsx

@@ -162,8 +162,12 @@ export const App: React.FC = () => {
       fetch("/embeddings/status")
         .then((res) => res.json())
         .then((data: EmbeddingStatus) => {
-          setEmbeddingStatus(data);
-          setEmbeddingRateLimit(String(data.rate_limit_per_sec));
+          setEmbeddingStatus((prev) => {
+            if (prev === null) {
+              setEmbeddingRateLimit(String(data.rate_limit_per_sec));
+            }
+            return data;
+          });
         })
         .catch(() => {
           /* ignore */
@@ -285,6 +289,7 @@ export const App: React.FC = () => {
       }
       const status: EmbeddingStatus = await res.json();
       setEmbeddingStatus(status);
+      setEmbeddingRateLimit(String(status.rate_limit_per_sec));
     } catch (e: any) {
       setError(e.message ?? String(e));
     } finally {
@@ -303,6 +308,7 @@ export const App: React.FC = () => {
       }
       const status: EmbeddingStatus = await res.json();
       setEmbeddingStatus(status);
+      setEmbeddingRateLimit(String(status.rate_limit_per_sec));
     } catch (e: any) {
       setError(e.message ?? String(e));
     } finally {
@@ -325,6 +331,31 @@ export const App: React.FC = () => {
       }
       const status: EmbeddingStatus = await res.json();
       setEmbeddingStatus(status);
+      setEmbeddingRateLimit(String(status.rate_limit_per_sec));
+    } catch (e: any) {
+      setError(e.message ?? String(e));
+    } finally {
+      setIsMutatingEmbeddingJob(false);
+    }
+  };
+
+  const clearAllEmbeddings = async () => {
+    const confirmed = window.confirm(
+      "Are you sure you want to delete all embeddings? This cannot be undone and semantic search will be empty until you embed again."
+    );
+    if (!confirmed) {
+      return;
+    }
+    setError(null);
+    try {
+      setIsMutatingEmbeddingJob(true);
+      const res = await fetch("/embeddings/clear", { method: "POST" });
+      if (!res.ok) {
+        const data = await res.json();
+        throw new Error(data.detail ?? "Failed to clear embeddings");
+      }
+      const status: EmbeddingStatus = await res.json();
+      setEmbeddingStatus(status);
     } catch (e: any) {
       setError(e.message ?? String(e));
     } finally {
@@ -481,6 +512,9 @@ export const App: React.FC = () => {
         <button className="button-secondary" onClick={updateEmbeddingRateLimit} disabled={isMutatingEmbeddingJob}>
           Update Speed
         </button>
+        <button className="button-secondary" onClick={clearAllEmbeddings} disabled={isMutatingEmbeddingJob}>
+          Delete All Embeddings
+        </button>
       </div>
       <div className="metrics-grid">
         <div className="metric">
@@ -492,6 +526,12 @@ export const App: React.FC = () => {
           <span className="value">
             {embeddingStatus ? `${embeddingStatus.progress_percent.toFixed(1)}% of your sites are embedded` : "Loading..."}
           </span>
+          <div className="progress-track" aria-label="Embedding progress">
+            <div
+              className="progress-fill"
+              style={{ width: `${Math.min(100, Math.max(0, embeddingStatus?.progress_percent ?? 0))}%` }}
+            />
+          </div>
         </div>
         <div className="metric">
           <span className="label">Embedded pages</span>

frontend/src/styles.css

@@ -203,6 +203,21 @@ button:disabled {
   margin-top: 0.15rem;
 }
 
+.progress-track {
+  margin-top: 0.45rem;
+  width: 100%;
+  height: 0.45rem;
+  border-radius: 999px;
+  background: rgba(148, 163, 184, 0.2);
+  overflow: hidden;
+}
+
+.progress-fill {
+  height: 100%;
+  background: linear-gradient(90deg, #6366f1, #a855f7);
+  transition: width 0.25s ease;
+}
+
 .badge {
   display: inline-flex;
   align-items: center;

product_prd.md

@@ -12,7 +12,7 @@ Building search engines is usually out of reach for a single engineer, but AI-as
 - **Live behavior**: allow search while indexing is running so new pages appear in results as soon as they are processed.
 - **Visibility**: provide a React dashboard that exposes crawl progress, queue depth, backpressure state, and job list.
 - **Local persistence**: store a lightweight snapshot of pages and jobs on disk so that search can resume after a restart.
-- **Embedding operations**: provide a separate UI mode to start/pause/resume embedding jobs with speed and max-page controls.
+- **Embedding operations**: provide a separate UI mode to control a single embedding engine (start/pause, speed, max-page scope) and reset embeddings when needed.
 - **AI-friendly design**: keep the crawler and indexer logic explicit and understandable, avoiding “black box” scraping libraries.
 
 #### Non-goals
@@ -47,11 +47,12 @@ Building search engines is usually out of reach for a single engineer, but AI-as
   - Reads directly from the current in-memory index so queries reflect pages as soon as they are indexed.
   - `GET /search/semantic?query=...` returns semantic matches from persisted page embeddings with similarity scores.
 
-- **Embedding job endpoints**
-  - `POST /embeddings/jobs/start` starts a manual embedding run with `{ rate_limit_per_sec, max_pages }`.
-  - `POST /embeddings/jobs/{job_id}/pause|resume` controls active embedding jobs.
-  - `POST /embeddings/jobs/{job_id}/rate-limit` updates embedding speed during runtime.
-  - `GET /embeddings/jobs` and `GET /embeddings/jobs/{job_id}` expose embedding progress and status.
+- **Embedding engine endpoints**
+  - `GET /embeddings/status` returns embedding progress and current engine state.
+  - `POST /embeddings/start` starts (or continues) embedding missing pages with `{ rate_limit_per_sec, max_pages }`.
+  - `POST /embeddings/pause` pauses active embedding work.
+  - `POST /embeddings/rate-limit` updates embedding speed during runtime.
+  - `POST /embeddings/clear` deletes all stored embeddings after explicit user confirmation in UI.
 
 - **Status/metrics endpoints**
   - `GET /jobs/{job_id}` returns job metadata and per-job statistics (status, processed URLs, queue depth estimate, backpressure state).
@@ -60,13 +61,13 @@ Building search engines is usually out of reach for a single engineer, but AI-as
 - **React UI**
   - **Index control panel**: origin+depth form, “Start Indexing” button, and active job id display.
   - **Search panel**: query input with side-by-side classical and semantic result tables.
-  - **Embeddings panel**: manual embedding controls (start/pause/resume, speed, max-page limit) and live progress cards.
+  - **Embeddings panel**: manual embedding controls (start/pause, speed, max-page limit, clear-all reset) and live progress cards.
   - **System dashboard**: live metrics cards and jobs table, auto-refreshing every few seconds.
 
 - **Persistence**
   - Store pages in a local SQLite database (via Python `sqlite3` stdlib) with columns `(url, origin_url, depth, title, body_snippet)`.
   - Store job metadata `(id, origin_url, max_depth, created_at, status)` in the same DB.
-  - Store semantic vectors in `page_embeddings` and embedding-job metadata in `embedding_jobs`.
+  - Store semantic vectors in `page_embeddings`; progress is derived from `pages` vs `page_embeddings`.
   - On startup, rebuild the in-memory index from stored pages and repopulate the job list from stored jobs.
 
 #### Constraints

recommendation.md

@@ -1,6 +1,6 @@
 ### Production deployment recommendations
 
-To harden this crawler and searcher for a real production environment, the first step would be to separate concerns into independent services: a crawl scheduler, a fleet of crawler workers, an indexing pipeline, a search API, and a UI/API gateway. The current single-process Python implementation can evolve into a **worker template**: wrap the crawler loop into a process that pulls crawl tasks from a durable queue (for example, Kafka, RabbitMQ, or a managed cloud queue), and writes fetched pages to a durable storage and indexing service. Since the project now supports both classical and semantic retrieval, the next production step is a hybrid search layer: keep lexical ranking in a dedicated search engine (such as OpenSearch/Elasticsearch or a managed search service) and move vector storage/retrieval to a scalable vector-capable backend. For persistence, replace the local SQLite database with a replicated database (PostgreSQL or similar) to hold crawl jobs, page metadata, embedding-job state, and operational rate-limit state, and back it with periodic backups and observability.
+To harden this crawler and searcher for a real production environment, the first step would be to separate concerns into independent services: a crawl scheduler, a fleet of crawler workers, an indexing pipeline, a search API, and a UI/API gateway. The current single-process Python implementation can evolve into a **worker template**: wrap the crawler loop into a process that pulls crawl tasks from a durable queue (for example, Kafka, RabbitMQ, or a managed cloud queue), and writes fetched pages to a durable storage and indexing service. Since the project now supports both classical and semantic retrieval, the next production step is a hybrid search layer: keep lexical ranking in a dedicated search engine (such as OpenSearch/Elasticsearch or a managed search service) and move vector storage/retrieval to a scalable vector-capable backend. For persistence, replace the local SQLite database with a replicated database (PostgreSQL or similar) to hold crawl jobs, page metadata, embedding coverage state, and operational rate-limit state, and back it with periodic backups and observability.
 
 On the operational side, the system should adopt **strong politeness and safety controls** before crawling the public internet. That includes honoring `robots.txt`, enforcing per-domain and global rate limits, respecting crawl budgets per domain, and providing a “kill switch” for misbehaving jobs. The services should run inside containers orchestrated by Kubernetes or another orchestrator, with metrics and tracing exported to Prometheus/Grafana or an equivalent stack. Authentication and authorization should be added around the APIs, and audit logging enabled for search queries if required. Finally, ranking can evolve from the current dual-mode baseline (classical + semantic) into a calibrated hybrid ranker (lexical + vector re-ranking), while still keeping an interpretable “debug view” so engineers and students can understand why particular URLs are returned for a query.