feat(search): switch to DuckDuckGo as default search provider with Google fallback

- Replace Google search as the default for web scraping
- Implement performDuckDuckGoSearchAndScrape() with robust HTML parsing
- Update manifest.json to include DuckDuckGo host permissions
- Add user preference toggle for DuckDuckGo-only mode in Options
- Update sidepanel to use unified SEARCH_AND_SCRAPE API
- Add Checkbox component for consistent UI patterns
- Resolves Google blocking issues by prioritizing DuckDuckGo

Author: justrach

bun.lock

@@ -4,6 +4,7 @@
     "": {
       "name": "jan-browser-extension",
       "dependencies": {
+        "@radix-ui/react-checkbox": "^1.1.6",
         "@radix-ui/react-label": "^2.1.7",
         "@radix-ui/react-popover": "^1.1.14",
         "@radix-ui/react-scroll-area": "^1.2.9",
@@ -192,10 +193,12 @@
 
     "@radix-ui/number": ["@radix-ui/[email protected]", "", {}, "sha512-MkKCwxlXTgz6CFoJx3pCwn07GKp36+aZyu/u2Ln2VrA5DcdyCZkASEDBTd8x5whTQQL5CiYf4prXKLcgQdv29g=="],
 
-    "@radix-ui/primitive": ["@radix-ui/[email protected].2", "", {}, "sha512-XnbHrrprsNqZKQhStrSwgRUQzoCI1glLzdw79xiZPoofhGICeZRSQ3dIxAKH1gb3OHfNf4d6f+vAv3kil2eggA=="],
+    "@radix-ui/primitive": ["@radix-ui/[email protected].3", "", {}, "sha512-JTF99U/6XIjCBo0wqkU5sK10glYe27MRRsfwoiq5zzOEZLHU3A3KCMa5X/azekYRCJ0HlwI0crAXS/5dEHTzDg=="],
 
     "@radix-ui/react-arrow": ["@radix-ui/[email protected]", "", { "dependencies": { "@radix-ui/react-primitive": "2.1.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-F+M1tLhO+mlQaOWspE8Wstg+z6PwxwRd8oQ8IXceWz92kfAmalTRf0EjrouQeo7QssEPfCn05B4Ihs1K9WQ/7w=="],
 
+    "@radix-ui/react-checkbox": ["@radix-ui/[email protected]", "", { "dependencies": { "@radix-ui/primitive": "1.1.3", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-presence": "1.1.5", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-controllable-state": "1.2.2", "@radix-ui/react-use-previous": "1.1.1", "@radix-ui/react-use-size": "1.1.1" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-wBbpv+NQftHDdG86Qc0pIyXk5IR3tM8Vd0nWLKDcX8nNn4nXFOFwsKuqw2okA/1D/mpaAkmuyndrPJTYDNZtFw=="],
+
     "@radix-ui/react-compose-refs": ["@radix-ui/[email protected]", "", { "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-z4eqJvfiNnFMHIIvXP3CY57y2WJs5g2v3X0zm9mEJkrkNv4rDxu+sg9Jh8EkXyeqBkB7SOcboo9dMVqhyrACIg=="],
 
     "@radix-ui/react-context": ["@radix-ui/[email protected]", "", { "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-jCi/QKUM2r1Ju5a3J64TH2A5SpKAgh0LpknyqdQ4m6DCV0xJ2HG1xARRwNGPQfi1SLdLWZ1OJz6F4OMBBNiGJA=="],
@@ -218,7 +221,7 @@
 
     "@radix-ui/react-portal": ["@radix-ui/[email protected]", "", { "dependencies": { "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-layout-effect": "1.1.1" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-bpIxvq03if6UNwXZ+HTK71JLh4APvnXntDc6XOX8UVq4XQOVl7lwok0AvIl+b8zgCw3fSaVTZMpAPPagXbKmHQ=="],
 
-    "@radix-ui/react-presence": ["@radix-ui/[email protected].4", "", { "dependencies": { "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-use-layout-effect": "1.1.1" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-ueDqRbdc4/bkaQT3GIpLQssRlFgWaL/U2z/S31qRwwLWoxHLgry3SIfCwhxeQNbirEUXFa+lq3RL3oBYXtcmIA=="],
+    "@radix-ui/react-presence": ["@radix-ui/[email protected].5", "", { "dependencies": { "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-use-layout-effect": "1.1.1" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-/jfEwNDdQVBCNvjkGit4h6pMOzq8bHkopq458dPt2lMjx+eBQUohZNG9A7DtO/O5ukSbxuaNGXMjHicgwy6rQQ=="],
 
     "@radix-ui/react-primitive": ["@radix-ui/[email protected]", "", { "dependencies": { "@radix-ui/react-slot": "1.2.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-m9gTwRkhy2lvCPe6QJp4d3G1TYEUHn/FzJUtq9MjH46an1wJU+GdoGC5VLof8RX8Ft/DlpshApkhswDLZzHIcQ=="],
 
@@ -762,7 +765,19 @@
 
     "@babel/helper-compilation-targets/semver": ["[email protected]", "", { "bin": "bin/semver.js" }, "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA=="],
 
-    "@radix-ui/react-switch/@radix-ui/primitive": ["@radix-ui/[email protected]", "", {}, "sha512-JTF99U/6XIjCBo0wqkU5sK10glYe27MRRsfwoiq5zzOEZLHU3A3KCMa5X/azekYRCJ0HlwI0crAXS/5dEHTzDg=="],
+    "@radix-ui/react-dismissable-layer/@radix-ui/primitive": ["@radix-ui/[email protected]", "", {}, "sha512-XnbHrrprsNqZKQhStrSwgRUQzoCI1glLzdw79xiZPoofhGICeZRSQ3dIxAKH1gb3OHfNf4d6f+vAv3kil2eggA=="],
+
+    "@radix-ui/react-popover/@radix-ui/primitive": ["@radix-ui/[email protected]", "", {}, "sha512-XnbHrrprsNqZKQhStrSwgRUQzoCI1glLzdw79xiZPoofhGICeZRSQ3dIxAKH1gb3OHfNf4d6f+vAv3kil2eggA=="],
+
+    "@radix-ui/react-popover/@radix-ui/react-presence": ["@radix-ui/[email protected]", "", { "dependencies": { "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-use-layout-effect": "1.1.1" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-ueDqRbdc4/bkaQT3GIpLQssRlFgWaL/U2z/S31qRwwLWoxHLgry3SIfCwhxeQNbirEUXFa+lq3RL3oBYXtcmIA=="],
+
+    "@radix-ui/react-scroll-area/@radix-ui/primitive": ["@radix-ui/[email protected]", "", {}, "sha512-XnbHrrprsNqZKQhStrSwgRUQzoCI1glLzdw79xiZPoofhGICeZRSQ3dIxAKH1gb3OHfNf4d6f+vAv3kil2eggA=="],
+
+    "@radix-ui/react-scroll-area/@radix-ui/react-presence": ["@radix-ui/[email protected]", "", { "dependencies": { "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-use-layout-effect": "1.1.1" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-ueDqRbdc4/bkaQT3GIpLQssRlFgWaL/U2z/S31qRwwLWoxHLgry3SIfCwhxeQNbirEUXFa+lq3RL3oBYXtcmIA=="],
+
+    "@radix-ui/react-tooltip/@radix-ui/primitive": ["@radix-ui/[email protected]", "", {}, "sha512-XnbHrrprsNqZKQhStrSwgRUQzoCI1glLzdw79xiZPoofhGICeZRSQ3dIxAKH1gb3OHfNf4d6f+vAv3kil2eggA=="],
+
+    "@radix-ui/react-tooltip/@radix-ui/react-presence": ["@radix-ui/[email protected]", "", { "dependencies": { "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-use-layout-effect": "1.1.1" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-ueDqRbdc4/bkaQT3GIpLQssRlFgWaL/U2z/S31qRwwLWoxHLgry3SIfCwhxeQNbirEUXFa+lq3RL3oBYXtcmIA=="],
 
     "chalk/supports-color": ["[email protected]", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw=="],
 

issue_google.md

@@ -0,0 +1,98 @@
+# Issue: Google SERP scraping is unreliable/intermittent
+
+## Summary
+- __Problem__: Our extension’s Google search scraping sometimes returns 0 results or stalls despite opening a SERP tab.
+- __Impact__: Blocks the MCP `search` tool and side panel features that depend on Google results; user-facing queries fail or degrade.
+- __Scope__: Background flow (`performGoogleSearchAndScrape()`), content script selectors/ready checks, and MCP bridge.
+
+## Environment
+- Chrome Extension (MV3) with background service worker: `src/background.js`
+- Content script scraping: `src/content.js`
+- MCP bridge/server (websocket): `mcp/search-server/src/index.ts`
+
+## Expected vs Actual
+- __Expected__: Given a query, open Google SERP and return N organic results (`numResults`, 1–10) + optional answer box.
+- __Actual__: Intermittently returns empty results or fails readiness; sometimes appears to stop on unrelated tabs.
+
+## Repro (minimal)
+1. Start dev processes: `npm run dev:all`.
+2. From MCP client, call `search` with a query and `numResults` (e.g., 5).
+3. Observe occasional 0 results or readiness timeouts, despite SERP loading.
+
+## What we tried (code references)
+- __Threaded numResults end-to-end__
+  - Bridge handler forwards `params.numResults` → `performGoogleSearchAndScrape()` in `src/background.js`.
+  - Content script respects `payload.numResults` in `SCRAPE_GOOGLE_SERP` (`src/content.js`).
+- __Realistic Google URLs__
+  - Added `oq`, `sourceid=chrome`, `ie=UTF-8`, `hl`, `gl`, `sclient=gws-wiz-serp` in `performGoogleSearchAndScrape()` (`src/background.js`).
+- __Readiness polling with jitter__
+  - `WAIT_FOR_SERP_READY` randomized interval; min results derived from `numResults` (`src/content.js` + `src/background.js`).
+- __Human-like interaction__
+  - `HUMANIZE_SERP` small smooth scrolls and random waits before scraping (`src/content.js`, invoked from `src/background.js`).
+- __Robust scraping path + debug mode__
+  - Primary selectors for organic results; fallback path; optional diagnostics (`SCRAPE_GOOGLE_SERP` in `src/content.js`).
+- __MCP bridge path__
+  - `connectMcpBridge()` → `performGoogleSearchAndScrape()`; returns `{answerBox, results}` (`src/background.js`, `mcp/search-server/src/index.ts`).
+
+## Observations
+- Google DOM/layout varies by locale/experiment; some pages delay organic nodes or interleave modules (Top stories, AI Overviews, People Also Ask), breaking early readiness assumptions.
+- Intermittent 0 organic results even when SERP visually loads (likely timing/selector drift).
+- Rare cases where active tab context/logs suggest focus on a non-SERP tab during scrape window.
+
+## Proposed next steps
+- __Add DuckDuckGo fallback__ (if Google readiness fails or results.length === 0):
+  - Open DDG SERP and scrape organic results with analogous handlers.
+  - New content handlers: `WAIT_FOR_DDG_READY`, `SCRAPE_DDG_SERP` (`src/content.js`).
+  - Gate by feature flag and surface source in the response payload.
+- __Selector hardening__ for Google:
+  - Expand organic container selectors and ignore ads/modules consistently.
+  - Add a second-chance readiness after minor scroll/short delay.
+- __Timeout/threshold tuning__:
+  - Slightly increase `readinessTimeoutMs` for cold start; minResults heuristic based on `numResults` and page modules.
+- __Better tab targeting__:
+  - Ensure scrape messages are sent only to the created SERP tab ID and verify URL origin before acting.
+- __Optional__: Introduce Google CSE HTTP fallback when running outside the browser (if policy allows and keys are available).
+
+## Implementation plan
+
+- __Permissions__
+  - Add DuckDuckGo host permission in `manifest.json`: `https://duckduckgo.com/*`.
+
+- __Background flow__ (`src/background.js`)
+  - Track and use the created SERP `tabId` exclusively for readiness/scrape messaging.
+  - Verify origin before acting using helpers like `isGoogleSerp(url)` / `isDuckDuckGoSerp(url)`.
+  - Control flow: try Google → if readiness fails or 0 results, fall back to DDG and return its results.
+  - Always include `{ source: 'google' | 'duckduckgo' }` in the returned payload, together with `results` and optional `answerBox`.
+  - Tune timeouts: allow a slightly higher `readinessTimeoutMs` on first try; keep jittered polling; add a second-chance readiness after a small scroll.
+
+- __Content script__ (`src/content.js`)
+  - Google hardening: broaden organic-result detection under the main results area and consistently ignore ads/modules (Top stories, PAA, AI Overviews, etc.).
+  - Add `WAIT_FOR_DDG_READY` and `SCRAPE_DDG_SERP` handlers with robust selectors that target organic result cards under the DDG results container, skipping sponsored/instant-answer blocks.
+  - Add a second-chance readiness probe after a short delay + minor scroll before declaring failure.
+  - Instrument debug logs and counts for readiness attempts, total results, and timing when debug mode is enabled.
+
+- __MCP bridge__ (`mcp/search-server/src/index.ts`)
+  - Ensure `numResults` threading and pass-through of `{ source, results, answerBox }` from background to MCP response.
+
+- __UI__ (`ui/sidepanel/App.jsx`)
+  - Surface the search source (Google/DDG) and basic diagnostics in debug mode.
+
+- __Docs & tests__
+  - Update `docs/SPEC*.md` on scraping sources and behavior.
+  - Add a manual smoke test checklist and capture sample logs to verify readiness/scrape timing.
+
+## Task checklist
+
+- [ ] Add `https://duckduckgo.com/*` to `manifest.json` host permissions.
+- [ ] Background: restrict messages to the created SERP tab; verify URL origin; implement fallback to DDG; return `{ source, results, answerBox }`.
+- [ ] Content: implement `WAIT_FOR_DDG_READY` and `SCRAPE_DDG_SERP`; harden Google selectors; add second-chance readiness and gentle scroll.
+- [ ] Timing: tune `readinessTimeoutMs` and polling jitter; enable a one-time extended timeout on cold start.
+- [ ] Debug: add diagnostics counters/timing and expose them behind a debug flag.
+- [ ] MCP: thread `numResults` and include `source` in responses.
+- [ ] UI: display search `source` and show debug diagnostics in dev mode.
+- [ ] Docs: update SPEC/ADR and add selector notes; add a manual smoke-test checklist.
+
+## Acceptance criteria
+  - With `numResults` in [1..10], at least 95% of calls return the requested number from Google OR fall back to DDG with the same count.
+  - No stalls; response includes source metadata: `{ source: 'google' | 'duckduckgo' }`.
+  - Debug mode shows counts and timing diagnostics for readiness and scrape.

manifest.json

@@ -26,7 +26,9 @@
   ],
   "host_permissions": [
     "https://*/*",
-    "http://*/*"
+    "http://*/*",
+    "https://duckduckgo.com/*",
+    "https://html.duckduckgo.com/*"
   ],
   "background": {
     "service_worker": "src/background.js",

package.json

@@ -22,6 +22,7 @@
   "author": "",
   "license": "Apache-2.0",
   "dependencies": {
+    "@radix-ui/react-checkbox": "^1.1.6",
     "@radix-ui/react-label": "^2.1.7",
     "@radix-ui/react-popover": "^1.1.14",
     "@radix-ui/react-scroll-area": "^1.2.9",