solution for homework

bundan sonrasi fazla olucak

Author: TalhaMudo

backend/__pycache__/app.cpython-314.pyc

@@ -17,6 +17,7 @@
 class IndexRequest(BaseModel):
     origin: str
     k: int
+    max_urls_to_visit: int | None = None
     rate_limit_per_sec: float | None = None
 
 
@@ -48,6 +49,7 @@ class JobDetailResponse(BaseModel):
     id: str
     origin_url: str
     max_depth: int
+    max_urls_to_visit: int | None = None
     created_at: str
     updated_at: str
     status: str
@@ -87,12 +89,15 @@ class MetricsResponse(BaseModel):
 async def index(request: IndexRequest) -> IndexResponse:
     if request.k < 0:
         raise HTTPException(status_code=400, detail="k must be non-negative")
+    if request.max_urls_to_visit is not None and request.max_urls_to_visit <= 0:
+        raise HTTPException(status_code=400, detail="max_urls_to_visit must be positive")
 
     job_id = str(uuid.uuid4())
     job = CrawlJob(
         id=job_id,
         origin_url=request.origin,
         max_depth=request.k,
+        max_urls_to_visit=request.max_urls_to_visit,
         created_at=datetime.utcnow(),
         status=JobStatus.PENDING,
     )
@@ -140,6 +145,7 @@ def _serialize_job(job_id: str) -> JobDetailResponse:
         id=job.id,
         origin_url=job.origin_url,
         max_depth=job.max_depth,
+        max_urls_to_visit=job.max_urls_to_visit,
         created_at=job.created_at.isoformat(),
         updated_at=job.updated_at.isoformat(),
         status=job.status.value,

backend/__pycache__/crawler.cpython-314.pyc

@@ -82,6 +82,7 @@ class CrawlContext:
     state_dirty: bool = False
     checkpoint_counter: int = 0
     checkpoint_ts: float = 0.0
+    max_reached_notified: bool = False
 
 
 class CrawlerService:
@@ -302,6 +303,15 @@ async def _worker(self, ctx: CrawlContext) -> None:
 
     async def _try_mark_visited(self, ctx: CrawlContext, url: str, depth: int) -> bool:
         async with self._lock:
+            if ctx.job.max_urls_to_visit is not None and len(ctx.visited) >= ctx.job.max_urls_to_visit:
+                if not ctx.max_reached_notified:
+                    self._append_event(
+                        ctx.job.id,
+                        "info",
+                        f"Max URLs reached ({ctx.job.max_urls_to_visit}); skipping remaining queue entries",
+                    )
+                    ctx.max_reached_notified = True
+                return False
             if url in ctx.visited:
                 self._append_event(ctx.job.id, "debug", "Skipped already-visited URL", url=url, depth=depth)
                 return False
@@ -337,6 +347,8 @@ async def _enqueue(self, ctx: CrawlContext, url: str, depth: int, origin_url: st
             return False
         if await self._already_visited(ctx, url):
             return False
+        if not await self._can_add_more_urls(ctx):
+            return False
         while True:
             if ctx.stop_requested or ctx.job.status != JobStatus.RUNNING:
                 return False
@@ -359,6 +371,12 @@ async def _enqueue(self, ctx: CrawlContext, url: str, depth: int, origin_url: st
                 self._update_global_stats()
                 await asyncio.sleep(0)
 
+    async def _can_add_more_urls(self, ctx: CrawlContext) -> bool:
+        if ctx.job.max_urls_to_visit is None:
+            return True
+        async with self._lock:
+            return (len(ctx.visited) + len(ctx.frontier)) < ctx.job.max_urls_to_visit
+
     async def _maybe_checkpoint(self, ctx: CrawlContext) -> None:
         if not ctx.state_dirty:
             return

backend/__pycache__/models.cpython-314.pyc

@@ -33,6 +33,7 @@ class CrawlJob:
     origin_url: str
     max_depth: int
     created_at: datetime
+    max_urls_to_visit: int | None = None
     rate_limit_per_sec: float = 1.0
     status: JobStatus = JobStatus.PENDING
     error_message: Optional[str] = None
@@ -52,6 +53,7 @@ def summarize_jobs(jobs: Dict[str, CrawlJob]) -> List[Dict[str, object]]:
                 "id": job.id,
                 "origin_url": job.origin_url,
                 "max_depth": job.max_depth,
+                "max_urls_to_visit": job.max_urls_to_visit,
                 "created_at": job.created_at.isoformat(),
                 "updated_at": job.updated_at.isoformat(),
                 "status": job.status.value,

backend/__pycache__/storage.cpython-314.pyc

@@ -35,6 +35,7 @@ def init_db() -> None:
                 id TEXT PRIMARY KEY,
                 origin_url TEXT NOT NULL,
                 max_depth INTEGER NOT NULL,
+                max_urls_to_visit INTEGER,
                 created_at TEXT NOT NULL,
                 updated_at TEXT NOT NULL,
                 status TEXT NOT NULL,
@@ -52,6 +53,7 @@ def init_db() -> None:
             """
         )
         _add_column_if_missing(cur, "jobs", "updated_at", "TEXT NOT NULL DEFAULT ''")
+        _add_column_if_missing(cur, "jobs", "max_urls_to_visit", "INTEGER")
         _add_column_if_missing(cur, "jobs", "rate_limit_per_sec", "REAL NOT NULL DEFAULT 1.0")
         _add_column_if_missing(cur, "jobs", "error_message", "TEXT")
         _add_column_if_missing(cur, "jobs", "processed_urls", "INTEGER NOT NULL DEFAULT 0")
@@ -168,6 +170,7 @@ def _upsert_job(cur: sqlite3.Cursor, job: CrawlJob) -> None:
             id,
             origin_url,
             max_depth,
+            max_urls_to_visit,
             created_at,
             updated_at,
             status,
@@ -182,12 +185,13 @@ def _upsert_job(cur: sqlite3.Cursor, job: CrawlJob) -> None:
             active_workers,
             backpressure_state
         )
-        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
         """,
         (
             job.id,
             job.origin_url,
             job.max_depth,
+            job.max_urls_to_visit,
             job.created_at.isoformat(),
             job.updated_at.isoformat(),
             job.status.value,
@@ -293,6 +297,7 @@ def load_jobs() -> List[CrawlJob]:
                 id,
                 origin_url,
                 max_depth,
+                max_urls_to_visit,
                 created_at,
                 updated_at,
                 status,
@@ -315,6 +320,7 @@ def load_jobs() -> List[CrawlJob]:
             job_id,
             origin_url,
             max_depth,
+            max_urls_to_visit,
             created_at,
             updated_at,
             status,
@@ -335,6 +341,7 @@ def load_jobs() -> List[CrawlJob]:
                     origin_url=str(origin_url),
                     max_depth=int(max_depth),
                     created_at=datetime.fromisoformat(str(created_at)),
+                    max_urls_to_visit=int(max_urls_to_visit) if max_urls_to_visit is not None else None,
                     status=JobStatus(str(status)),
                     rate_limit_per_sec=float(rate_limit_per_sec or 1.0),
                     error_message=str(error_message) if error_message is not None else None,

backend/app.py

@@ -4,6 +4,7 @@ type JobSummary = {
   id: string;
   origin_url: string;
   max_depth: number;
+  max_urls_to_visit?: number | null;
   created_at: string;
   updated_at: string;
   status: string;
@@ -25,6 +26,7 @@ type JobDetail = {
   id: string;
   origin_url: string;
   max_depth: number;
+  max_urls_to_visit?: number | null;
   created_at: string;
   updated_at: string;
   status: string;
@@ -74,6 +76,7 @@ export const App: React.FC = () => {
   const [viewMode, setViewMode] = useState<"crawler" | "search">("crawler");
   const [origin, setOrigin] = useState("");
   const [depth, setDepth] = useState(2);
+  const [maxUrlsToVisit, setMaxUrlsToVisit] = useState("500");
   const [rateLimit, setRateLimit] = useState(1.0);
   const [currentJobId, setCurrentJobId] = useState<string | null>(null);
   const [selectedJobId, setSelectedJobId] = useState<string | null>(null);
@@ -141,7 +144,12 @@ export const App: React.FC = () => {
       const res = await fetch("/index", {
         method: "POST",
         headers: { "Content-Type": "application/json" },
-        body: JSON.stringify({ origin, k: depth, rate_limit_per_sec: rateLimit })
+        body: JSON.stringify({
+          origin,
+          k: depth,
+          max_urls_to_visit: maxUrlsToVisit ? Number(maxUrlsToVisit) : null,
+          rate_limit_per_sec: rateLimit,
+        }),
       });
       if (!res.ok) {
         const data = await res.json();
@@ -219,23 +227,34 @@ export const App: React.FC = () => {
   };
 
   const searchPanel = (
-    <section className="panel panel-search">
-      <h2>Search</h2>
+    <section className="panel panel-search panel-search-upgraded">
+      <div className="search-header">
+        <h2>
+          <span className="search-icon" aria-hidden="true">
+            🔎
+          </span>{" "}
+          Search
+        </h2>
+        <p className="search-subtitle">Find relevant URLs in pages already indexed by active or finished crawlers.</p>
+      </div>
       <label className="field">
         <span>Query</span>
         <input
+          className="search-input"
           type="text"
           placeholder="search terms"
           value={query}
           onChange={(e) => setQuery(e.target.value)}
         />
       </label>
-      <button onClick={runSearch} disabled={!query}>
-        Search Indexed Pages
-      </button>
-      <button onClick={runFeelingLucky} disabled={!query}>
-        I'm Feeling Lucky
-      </button>
+      <div className="search-actions">
+        <button onClick={runSearch} disabled={!query}>
+          Search Indexed Pages
+        </button>
+        <button className="button-secondary" onClick={runFeelingLucky} disabled={!query}>
+          I'm Feeling Lucky
+        </button>
+      </div>
       <div className="results">
         {results.length === 0 ? (
           <p className="hint">No results yet. Try searching after indexing.</p>
@@ -321,6 +340,15 @@ export const App: React.FC = () => {
                 onChange={(e) => setDepth(Number(e.target.value))}
               />
             </label>
+            <label className="field">
+              <span>Max URLs to Visit</span>
+              <input
+                type="number"
+                min={1}
+                value={maxUrlsToVisit}
+                onChange={(e) => setMaxUrlsToVisit(e.target.value)}
+              />
+            </label>
             <label className="field">
               <span>
                 Crawler Speed (req/s)
@@ -438,6 +466,7 @@ export const App: React.FC = () => {
                         <div className="job-card-url">{job.origin_url}</div>
                         <div className="job-card-meta">
                           <span>Depth: {job.max_depth}</span>
+                          <span>Max URLs: {job.max_urls_to_visit ?? "unbounded"}</span>
                           <span>Processed: {job.processed_urls}</span>
                           <span>Queue: {job.queued_urls}</span>
                           <span>{new Date(job.updated_at).toLocaleTimeString()}</span>
@@ -535,6 +564,10 @@ export const App: React.FC = () => {
                     <span className="label">Rate Limit</span>
                     <span className="value">{selectedJob.rate_limit_per_sec.toFixed(1)} req/s</span>
                   </div>
+                  <div className="metric">
+                    <span className="label">Max URLs</span>
+                    <span className="value">{selectedJob.max_urls_to_visit ?? "unbounded"}</span>
+                  </div>
                   <div className="metric">
                     <span className="label">Visited</span>
                     <span className="value">{selectedJob.visited_count}</span>

backend/crawler.py

@@ -126,6 +126,8 @@ input {
   border: 1px solid rgba(148, 163, 184, 0.6);
   background: rgba(15, 23, 42, 0.9);
   color: #f9fafb;
+  line-height: 1.25rem;
+  font-size: 0.95rem;
 }
 
 input:focus {
@@ -134,6 +136,12 @@ input:focus {
   box-shadow: 0 0 0 1px rgba(99, 102, 241, 0.4);
 }
 
+input::placeholder {
+  color: #94a3b8;
+  opacity: 1;
+  line-height: 1.25rem;
+}
+
 button {
   padding: 0.5rem 0.9rem;
   border-radius: 999px;
@@ -145,6 +153,11 @@ button {
   margin-top: 0.25rem;
 }
 
+.button-secondary {
+  background: rgba(30, 41, 59, 0.9);
+  border: 1px solid rgba(148, 163, 184, 0.35);
+}
+
 button:disabled {
   opacity: 0.5;
   cursor: not-allowed;
@@ -246,14 +259,52 @@ button:disabled {
   min-height: 0;
 }
 
+.panel-search-upgraded {
+  background: linear-gradient(165deg, rgba(16, 33, 74, 0.84), rgba(9, 18, 44, 0.84));
+  border-color: rgba(99, 102, 241, 0.25);
+}
+
+.search-header {
+  margin-bottom: 0.65rem;
+}
+
+.search-header h2 {
+  margin-bottom: 0.35rem;
+}
+
+.search-icon {
+  filter: drop-shadow(0 4px 8px rgba(79, 70, 229, 0.35));
+}
+
+.search-subtitle {
+  margin: 0;
+  font-size: 0.9rem;
+  color: #c7d2fe;
+}
+
+.search-input {
+  height: 2.6rem;
+}
+
+.search-actions {
+  display: flex;
+  gap: 0.6rem;
+  margin-bottom: 0.35rem;
+}
+
+.search-actions button {
+  flex: 1;
+  margin-top: 0;
+}
+
 .panel-job-detail {
   margin-bottom: 1rem;
 }
 
 .results {
   margin-top: 0.75rem;
   flex: 1;
-  min-height: 0;
+  min-height: 140px;
   overflow: auto;
   border: 1px solid rgba(148, 163, 184, 0.18);
   border-radius: 0.75rem;
@@ -476,5 +527,9 @@ td a {
     width: 100%;
     justify-content: center;
   }
+
+  .search-actions {
+    flex-direction: column;
+  }
 }