skills
This commit is contained in:
Thomas G. Lopes
@@ -0,0 +1 @@
|
||||
node_modules/
|
||||
@@ -0,0 +1,79 @@
|
||||
---
|
||||
name: brave-search
|
||||
description: Web search and content extraction via Brave Search API. Use for searching documentation, facts, or any web content. Lightweight, no browser required.
|
||||
---
|
||||
|
||||
# Brave Search
|
||||
|
||||
Web search and content extraction using the official Brave Search API. No browser required.
|
||||
|
||||
## Setup
|
||||
|
||||
Requires a Brave Search API account with a free subscription. A credit card is required to create the free subscription (you won't be charged).
|
||||
|
||||
1. Create an account at https://api-dashboard.search.brave.com/register
|
||||
2. Create a "Free AI" subscription
|
||||
3. Create an API key for the subscription
|
||||
4. Add to your shell profile (`~/.profile` or `~/.zprofile` for zsh):
|
||||
```bash
|
||||
export BRAVE_API_KEY="your-api-key-here"
|
||||
```
|
||||
5. Install dependencies (run once):
|
||||
```bash
|
||||
cd {baseDir}
|
||||
npm install
|
||||
```
|
||||
|
||||
## Search
|
||||
|
||||
```bash
|
||||
{baseDir}/search.js "query" # Basic search (5 results)
|
||||
{baseDir}/search.js "query" -n 10 # More results (max 20)
|
||||
{baseDir}/search.js "query" --content # Include page content as markdown
|
||||
{baseDir}/search.js "query" --freshness pw # Results from last week
|
||||
{baseDir}/search.js "query" --freshness 2024-01-01to2024-06-30 # Date range
|
||||
{baseDir}/search.js "query" --country DE # Results from Germany
|
||||
{baseDir}/search.js "query" -n 3 --content # Combined options
|
||||
```
|
||||
|
||||
### Options
|
||||
|
||||
- `-n <num>` - Number of results (default: 5, max: 20)
|
||||
- `--content` - Fetch and include page content as markdown
|
||||
- `--country <code>` - Two-letter country code (default: US)
|
||||
- `--freshness <period>` - Filter by time:
|
||||
- `pd` - Past day (24 hours)
|
||||
- `pw` - Past week
|
||||
- `pm` - Past month
|
||||
- `py` - Past year
|
||||
- `YYYY-MM-DDtoYYYY-MM-DD` - Custom date range
|
||||
|
||||
## Extract Page Content
|
||||
|
||||
```bash
|
||||
{baseDir}/content.js https://example.com/article
|
||||
```
|
||||
|
||||
Fetches a URL and extracts readable content as markdown.
|
||||
|
||||
## Output Format
|
||||
|
||||
```
|
||||
--- Result 1 ---
|
||||
Title: Page Title
|
||||
Link: https://example.com/page
|
||||
Age: 2 days ago
|
||||
Snippet: Description from search results
|
||||
Content: (if --content flag used)
|
||||
Markdown content extracted from the page...
|
||||
|
||||
--- Result 2 ---
|
||||
...
|
||||
```
|
||||
|
||||
## When to Use
|
||||
|
||||
- Searching for documentation or API references
|
||||
- Looking up facts or current information
|
||||
- Fetching content from specific URLs
|
||||
- Any task requiring web search without interactive browsing
|
||||
+86
@@ -0,0 +1,86 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { Readability } from "@mozilla/readability";
|
||||
import { JSDOM } from "jsdom";
|
||||
import TurndownService from "turndown";
|
||||
import { gfm } from "turndown-plugin-gfm";
|
||||
|
||||
const url = process.argv[2];
|
||||
|
||||
if (!url) {
|
||||
console.log("Usage: content.js <url>");
|
||||
console.log("\nExtracts readable content from a webpage as markdown.");
|
||||
console.log("\nExamples:");
|
||||
console.log(" content.js https://example.com/article");
|
||||
console.log(" content.js https://doc.rust-lang.org/book/ch04-01-what-is-ownership.html");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
function htmlToMarkdown(html) {
|
||||
const turndown = new TurndownService({ headingStyle: "atx", codeBlockStyle: "fenced" });
|
||||
turndown.use(gfm);
|
||||
turndown.addRule("removeEmptyLinks", {
|
||||
filter: (node) => node.nodeName === "A" && !node.textContent?.trim(),
|
||||
replacement: () => "",
|
||||
});
|
||||
return turndown
|
||||
.turndown(html)
|
||||
.replace(/\[\\?\[\s*\\?\]\]\([^)]*\)/g, "")
|
||||
.replace(/ +/g, " ")
|
||||
.replace(/\s+,/g, ",")
|
||||
.replace(/\s+\./g, ".")
|
||||
.replace(/\n{3,}/g, "\n\n")
|
||||
.trim();
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
headers: {
|
||||
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
},
|
||||
signal: AbortSignal.timeout(15000),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
console.error(`HTTP ${response.status}: ${response.statusText}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const html = await response.text();
|
||||
const dom = new JSDOM(html, { url });
|
||||
const reader = new Readability(dom.window.document);
|
||||
const article = reader.parse();
|
||||
|
||||
if (article && article.content) {
|
||||
if (article.title) {
|
||||
console.log(`# ${article.title}\n`);
|
||||
}
|
||||
console.log(htmlToMarkdown(article.content));
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// Fallback: try to extract main content
|
||||
const fallbackDoc = new JSDOM(html, { url });
|
||||
const body = fallbackDoc.window.document;
|
||||
body.querySelectorAll("script, style, noscript, nav, header, footer, aside").forEach(el => el.remove());
|
||||
|
||||
const title = body.querySelector("title")?.textContent?.trim();
|
||||
const main = body.querySelector("main, article, [role='main'], .content, #content") || body.body;
|
||||
|
||||
if (title) {
|
||||
console.log(`# ${title}\n`);
|
||||
}
|
||||
|
||||
const text = main?.innerHTML || "";
|
||||
if (text.trim().length > 100) {
|
||||
console.log(htmlToMarkdown(text));
|
||||
} else {
|
||||
console.error("Could not extract readable content from this page.");
|
||||
process.exit(1);
|
||||
}
|
||||
} catch (e) {
|
||||
console.error(`Error: ${e.message}`);
|
||||
process.exit(1);
|
||||
}
|
||||
@@ -0,0 +1,621 @@
|
||||
{
|
||||
"name": "brave-search",
|
||||
"version": "1.0.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "brave-search",
|
||||
"version": "1.0.0",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@mozilla/readability": "^0.6.0",
|
||||
"jsdom": "^27.0.1",
|
||||
"turndown": "^7.2.2",
|
||||
"turndown-plugin-gfm": "^1.0.2"
|
||||
}
|
||||
},
|
||||
"node_modules/@asamuzakjp/css-color": {
|
||||
"version": "4.1.0",
|
||||
"resolved": "https://registry.npmjs.org/@asamuzakjp/css-color/-/css-color-4.1.0.tgz",
|
||||
"integrity": "sha512-9xiBAtLn4aNsa4mDnpovJvBn72tNEIACyvlqaNJ+ADemR+yeMJWnBudOi2qGDviJa7SwcDOU/TRh5dnET7qk0w==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@csstools/css-calc": "^2.1.4",
|
||||
"@csstools/css-color-parser": "^3.1.0",
|
||||
"@csstools/css-parser-algorithms": "^3.0.5",
|
||||
"@csstools/css-tokenizer": "^3.0.4",
|
||||
"lru-cache": "^11.2.2"
|
||||
}
|
||||
},
|
||||
"node_modules/@asamuzakjp/dom-selector": {
|
||||
"version": "6.7.5",
|
||||
"resolved": "https://registry.npmjs.org/@asamuzakjp/dom-selector/-/dom-selector-6.7.5.tgz",
|
||||
"integrity": "sha512-Eks6dY8zau4m4wNRQjRVaKQRTalNcPcBvU1ZQ35w5kKRk1gUeNCkVLsRiATurjASTp3TKM4H10wsI50nx3NZdw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@asamuzakjp/nwsapi": "^2.3.9",
|
||||
"bidi-js": "^1.0.3",
|
||||
"css-tree": "^3.1.0",
|
||||
"is-potential-custom-element-name": "^1.0.1",
|
||||
"lru-cache": "^11.2.2"
|
||||
}
|
||||
},
|
||||
"node_modules/@asamuzakjp/nwsapi": {
|
||||
"version": "2.3.9",
|
||||
"resolved": "https://registry.npmjs.org/@asamuzakjp/nwsapi/-/nwsapi-2.3.9.tgz",
|
||||
"integrity": "sha512-n8GuYSrI9bF7FFZ/SjhwevlHc8xaVlb/7HmHelnc/PZXBD2ZR49NnN9sMMuDdEGPeeRQ5d0hqlSlEpgCX3Wl0Q==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@csstools/color-helpers": {
|
||||
"version": "5.1.0",
|
||||
"resolved": "https://registry.npmjs.org/@csstools/color-helpers/-/color-helpers-5.1.0.tgz",
|
||||
"integrity": "sha512-S11EXWJyy0Mz5SYvRmY8nJYTFFd1LCNV+7cXyAgQtOOuzb4EsgfqDufL+9esx72/eLhsRdGZwaldu/h+E4t4BA==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/csstools"
|
||||
},
|
||||
{
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/csstools"
|
||||
}
|
||||
],
|
||||
"license": "MIT-0",
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/@csstools/css-calc": {
|
||||
"version": "2.1.4",
|
||||
"resolved": "https://registry.npmjs.org/@csstools/css-calc/-/css-calc-2.1.4.tgz",
|
||||
"integrity": "sha512-3N8oaj+0juUw/1H3YwmDDJXCgTB1gKU6Hc/bB502u9zR0q2vd786XJH9QfrKIEgFlZmhZiq6epXl4rHqhzsIgQ==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/csstools"
|
||||
},
|
||||
{
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/csstools"
|
||||
}
|
||||
],
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@csstools/css-parser-algorithms": "^3.0.5",
|
||||
"@csstools/css-tokenizer": "^3.0.4"
|
||||
}
|
||||
},
|
||||
"node_modules/@csstools/css-color-parser": {
|
||||
"version": "3.1.0",
|
||||
"resolved": "https://registry.npmjs.org/@csstools/css-color-parser/-/css-color-parser-3.1.0.tgz",
|
||||
"integrity": "sha512-nbtKwh3a6xNVIp/VRuXV64yTKnb1IjTAEEh3irzS+HkKjAOYLTGNb9pmVNntZ8iVBHcWDA2Dof0QtPgFI1BaTA==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/csstools"
|
||||
},
|
||||
{
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/csstools"
|
||||
}
|
||||
],
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@csstools/color-helpers": "^5.1.0",
|
||||
"@csstools/css-calc": "^2.1.4"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@csstools/css-parser-algorithms": "^3.0.5",
|
||||
"@csstools/css-tokenizer": "^3.0.4"
|
||||
}
|
||||
},
|
||||
"node_modules/@csstools/css-parser-algorithms": {
|
||||
"version": "3.0.5",
|
||||
"resolved": "https://registry.npmjs.org/@csstools/css-parser-algorithms/-/css-parser-algorithms-3.0.5.tgz",
|
||||
"integrity": "sha512-DaDeUkXZKjdGhgYaHNJTV9pV7Y9B3b644jCLs9Upc3VeNGg6LWARAT6O+Q+/COo+2gg/bM5rhpMAtf70WqfBdQ==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/csstools"
|
||||
},
|
||||
{
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/csstools"
|
||||
}
|
||||
],
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@csstools/css-tokenizer": "^3.0.4"
|
||||
}
|
||||
},
|
||||
"node_modules/@csstools/css-syntax-patches-for-csstree": {
|
||||
"version": "1.0.20",
|
||||
"resolved": "https://registry.npmjs.org/@csstools/css-syntax-patches-for-csstree/-/css-syntax-patches-for-csstree-1.0.20.tgz",
|
||||
"integrity": "sha512-8BHsjXfSciZxjmHQOuVdW2b8WLUPts9a+mfL13/PzEviufUEW2xnvQuOlKs9dRBHgRqJ53SF/DUoK9+MZk72oQ==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/csstools"
|
||||
},
|
||||
{
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/csstools"
|
||||
}
|
||||
],
|
||||
"license": "MIT-0",
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/@csstools/css-tokenizer": {
|
||||
"version": "3.0.4",
|
||||
"resolved": "https://registry.npmjs.org/@csstools/css-tokenizer/-/css-tokenizer-3.0.4.tgz",
|
||||
"integrity": "sha512-Vd/9EVDiu6PPJt9yAh6roZP6El1xHrdvIVGjyBsHR0RYwNHgL7FJPyIIW4fANJNG6FtyZfvlRPpFI4ZM/lubvw==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/csstools"
|
||||
},
|
||||
{
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/csstools"
|
||||
}
|
||||
],
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/@mixmark-io/domino": {
|
||||
"version": "2.2.0",
|
||||
"resolved": "https://registry.npmjs.org/@mixmark-io/domino/-/domino-2.2.0.tgz",
|
||||
"integrity": "sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw==",
|
||||
"license": "BSD-2-Clause"
|
||||
},
|
||||
"node_modules/@mozilla/readability": {
|
||||
"version": "0.6.0",
|
||||
"resolved": "https://registry.npmjs.org/@mozilla/readability/-/readability-0.6.0.tgz",
|
||||
"integrity": "sha512-juG5VWh4qAivzTAeMzvY9xs9HY5rAcr2E4I7tiSSCokRFi7XIZCAu92ZkSTsIj1OPceCifL3cpfteP3pDT9/QQ==",
|
||||
"license": "Apache-2.0",
|
||||
"engines": {
|
||||
"node": ">=14.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/agent-base": {
|
||||
"version": "7.1.4",
|
||||
"resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz",
|
||||
"integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">= 14"
|
||||
}
|
||||
},
|
||||
"node_modules/bidi-js": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmjs.org/bidi-js/-/bidi-js-1.0.3.tgz",
|
||||
"integrity": "sha512-RKshQI1R3YQ+n9YJz2QQ147P66ELpa1FQEg20Dk8oW9t2KgLbpDLLp9aGZ7y8WHSshDknG0bknqGw5/tyCs5tw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"require-from-string": "^2.0.2"
|
||||
}
|
||||
},
|
||||
"node_modules/css-tree": {
|
||||
"version": "3.1.0",
|
||||
"resolved": "https://registry.npmjs.org/css-tree/-/css-tree-3.1.0.tgz",
|
||||
"integrity": "sha512-0eW44TGN5SQXU1mWSkKwFstI/22X2bG1nYzZTYMAWjylYURhse752YgbE4Cx46AC+bAvI+/dYTPRk1LqSUnu6w==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"mdn-data": "2.12.2",
|
||||
"source-map-js": "^1.0.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^10 || ^12.20.0 || ^14.13.0 || >=15.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/cssstyle": {
|
||||
"version": "5.3.3",
|
||||
"resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-5.3.3.tgz",
|
||||
"integrity": "sha512-OytmFH+13/QXONJcC75QNdMtKpceNk3u8ThBjyyYjkEcy/ekBwR1mMAuNvi3gdBPW3N5TlCzQ0WZw8H0lN/bDw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@asamuzakjp/css-color": "^4.0.3",
|
||||
"@csstools/css-syntax-patches-for-csstree": "^1.0.14",
|
||||
"css-tree": "^3.1.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20"
|
||||
}
|
||||
},
|
||||
"node_modules/data-urls": {
|
||||
"version": "6.0.0",
|
||||
"resolved": "https://registry.npmjs.org/data-urls/-/data-urls-6.0.0.tgz",
|
||||
"integrity": "sha512-BnBS08aLUM+DKamupXs3w2tJJoqU+AkaE/+6vQxi/G/DPmIZFJJp9Dkb1kM03AZx8ADehDUZgsNxju3mPXZYIA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"whatwg-mimetype": "^4.0.0",
|
||||
"whatwg-url": "^15.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20"
|
||||
}
|
||||
},
|
||||
"node_modules/debug": {
|
||||
"version": "4.4.3",
|
||||
"resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz",
|
||||
"integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"ms": "^2.1.3"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=6.0"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"supports-color": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/decimal.js": {
|
||||
"version": "10.6.0",
|
||||
"resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.6.0.tgz",
|
||||
"integrity": "sha512-YpgQiITW3JXGntzdUmyUR1V812Hn8T1YVXhCu+wO3OpS4eU9l4YdD3qjyiKdV6mvV29zapkMeD390UVEf2lkUg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/entities": {
|
||||
"version": "6.0.1",
|
||||
"resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz",
|
||||
"integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==",
|
||||
"license": "BSD-2-Clause",
|
||||
"engines": {
|
||||
"node": ">=0.12"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/fb55/entities?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/html-encoding-sniffer": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-4.0.0.tgz",
|
||||
"integrity": "sha512-Y22oTqIU4uuPgEemfz7NDJz6OeKf12Lsu+QC+s3BVpda64lTiMYCyGwg5ki4vFxkMwQdeZDl2adZoqUgdFuTgQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"whatwg-encoding": "^3.1.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/http-proxy-agent": {
|
||||
"version": "7.0.2",
|
||||
"resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz",
|
||||
"integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"agent-base": "^7.1.0",
|
||||
"debug": "^4.3.4"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 14"
|
||||
}
|
||||
},
|
||||
"node_modules/https-proxy-agent": {
|
||||
"version": "7.0.6",
|
||||
"resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz",
|
||||
"integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"agent-base": "^7.1.2",
|
||||
"debug": "4"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 14"
|
||||
}
|
||||
},
|
||||
"node_modules/iconv-lite": {
|
||||
"version": "0.6.3",
|
||||
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz",
|
||||
"integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"safer-buffer": ">= 2.1.2 < 3.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/is-potential-custom-element-name": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz",
|
||||
"integrity": "sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/jsdom": {
|
||||
"version": "27.0.1",
|
||||
"resolved": "https://registry.npmjs.org/jsdom/-/jsdom-27.0.1.tgz",
|
||||
"integrity": "sha512-SNSQteBL1IlV2zqhwwolaG9CwhIhTvVHWg3kTss/cLE7H/X4644mtPQqYvCfsSrGQWt9hSZcgOXX8bOZaMN+kA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@asamuzakjp/dom-selector": "^6.7.2",
|
||||
"cssstyle": "^5.3.1",
|
||||
"data-urls": "^6.0.0",
|
||||
"decimal.js": "^10.6.0",
|
||||
"html-encoding-sniffer": "^4.0.0",
|
||||
"http-proxy-agent": "^7.0.2",
|
||||
"https-proxy-agent": "^7.0.6",
|
||||
"is-potential-custom-element-name": "^1.0.1",
|
||||
"parse5": "^8.0.0",
|
||||
"rrweb-cssom": "^0.8.0",
|
||||
"saxes": "^6.0.0",
|
||||
"symbol-tree": "^3.2.4",
|
||||
"tough-cookie": "^6.0.0",
|
||||
"w3c-xmlserializer": "^5.0.0",
|
||||
"webidl-conversions": "^8.0.0",
|
||||
"whatwg-encoding": "^3.1.1",
|
||||
"whatwg-mimetype": "^4.0.0",
|
||||
"whatwg-url": "^15.1.0",
|
||||
"ws": "^8.18.3",
|
||||
"xml-name-validator": "^5.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"canvas": "^3.0.0"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"canvas": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/lru-cache": {
|
||||
"version": "11.2.4",
|
||||
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.2.4.tgz",
|
||||
"integrity": "sha512-B5Y16Jr9LB9dHVkh6ZevG+vAbOsNOYCX+sXvFWFu7B3Iz5mijW3zdbMyhsh8ANd2mSWBYdJgnqi+mL7/LrOPYg==",
|
||||
"license": "BlueOak-1.0.0",
|
||||
"engines": {
|
||||
"node": "20 || >=22"
|
||||
}
|
||||
},
|
||||
"node_modules/mdn-data": {
|
||||
"version": "2.12.2",
|
||||
"resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.12.2.tgz",
|
||||
"integrity": "sha512-IEn+pegP1aManZuckezWCO+XZQDplx1366JoVhTpMpBB1sPey/SbveZQUosKiKiGYjg1wH4pMlNgXbCiYgihQA==",
|
||||
"license": "CC0-1.0"
|
||||
},
|
||||
"node_modules/ms": {
|
||||
"version": "2.1.3",
|
||||
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
|
||||
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/parse5": {
|
||||
"version": "8.0.0",
|
||||
"resolved": "https://registry.npmjs.org/parse5/-/parse5-8.0.0.tgz",
|
||||
"integrity": "sha512-9m4m5GSgXjL4AjumKzq1Fgfp3Z8rsvjRNbnkVwfu2ImRqE5D0LnY2QfDen18FSY9C573YU5XxSapdHZTZ2WolA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"entities": "^6.0.0"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/inikulin/parse5?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/punycode": {
|
||||
"version": "2.3.1",
|
||||
"resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz",
|
||||
"integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=6"
|
||||
}
|
||||
},
|
||||
"node_modules/require-from-string": {
|
||||
"version": "2.0.2",
|
||||
"resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz",
|
||||
"integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/rrweb-cssom": {
|
||||
"version": "0.8.0",
|
||||
"resolved": "https://registry.npmjs.org/rrweb-cssom/-/rrweb-cssom-0.8.0.tgz",
|
||||
"integrity": "sha512-guoltQEx+9aMf2gDZ0s62EcV8lsXR+0w8915TC3ITdn2YueuNjdAYh/levpU9nFaoChh9RUS5ZdQMrKfVEN9tw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/safer-buffer": {
|
||||
"version": "2.1.2",
|
||||
"resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
|
||||
"integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/saxes": {
|
||||
"version": "6.0.0",
|
||||
"resolved": "https://registry.npmjs.org/saxes/-/saxes-6.0.0.tgz",
|
||||
"integrity": "sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA==",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"xmlchars": "^2.2.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=v12.22.7"
|
||||
}
|
||||
},
|
||||
"node_modules/source-map-js": {
|
||||
"version": "1.2.1",
|
||||
"resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz",
|
||||
"integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==",
|
||||
"license": "BSD-3-Clause",
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/symbol-tree": {
|
||||
"version": "3.2.4",
|
||||
"resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz",
|
||||
"integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/tldts": {
|
||||
"version": "7.0.19",
|
||||
"resolved": "https://registry.npmjs.org/tldts/-/tldts-7.0.19.tgz",
|
||||
"integrity": "sha512-8PWx8tvC4jDB39BQw1m4x8y5MH1BcQ5xHeL2n7UVFulMPH/3Q0uiamahFJ3lXA0zO2SUyRXuVVbWSDmstlt9YA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"tldts-core": "^7.0.19"
|
||||
},
|
||||
"bin": {
|
||||
"tldts": "bin/cli.js"
|
||||
}
|
||||
},
|
||||
"node_modules/tldts-core": {
|
||||
"version": "7.0.19",
|
||||
"resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-7.0.19.tgz",
|
||||
"integrity": "sha512-lJX2dEWx0SGH4O6p+7FPwYmJ/bu1JbcGJ8RLaG9b7liIgZ85itUVEPbMtWRVrde/0fnDPEPHW10ZsKW3kVsE9A==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/tough-cookie": {
|
||||
"version": "6.0.0",
|
||||
"resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-6.0.0.tgz",
|
||||
"integrity": "sha512-kXuRi1mtaKMrsLUxz3sQYvVl37B0Ns6MzfrtV5DvJceE9bPyspOqk9xxv7XbZWcfLWbFmm997vl83qUWVJA64w==",
|
||||
"license": "BSD-3-Clause",
|
||||
"dependencies": {
|
||||
"tldts": "^7.0.5"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=16"
|
||||
}
|
||||
},
|
||||
"node_modules/tr46": {
|
||||
"version": "6.0.0",
|
||||
"resolved": "https://registry.npmjs.org/tr46/-/tr46-6.0.0.tgz",
|
||||
"integrity": "sha512-bLVMLPtstlZ4iMQHpFHTR7GAGj2jxi8Dg0s2h2MafAE4uSWF98FC/3MomU51iQAMf8/qDUbKWf5GxuvvVcXEhw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"punycode": "^2.3.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20"
|
||||
}
|
||||
},
|
||||
"node_modules/turndown": {
|
||||
"version": "7.2.2",
|
||||
"resolved": "https://registry.npmjs.org/turndown/-/turndown-7.2.2.tgz",
|
||||
"integrity": "sha512-1F7db8BiExOKxjSMU2b7if62D/XOyQyZbPKq/nUwopfgnHlqXHqQ0lvfUTeUIr1lZJzOPFn43dODyMSIfvWRKQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@mixmark-io/domino": "^2.2.0"
|
||||
}
|
||||
},
|
||||
"node_modules/turndown-plugin-gfm": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/turndown-plugin-gfm/-/turndown-plugin-gfm-1.0.2.tgz",
|
||||
"integrity": "sha512-vwz9tfvF7XN/jE0dGoBei3FXWuvll78ohzCZQuOb+ZjWrs3a0XhQVomJEb2Qh4VHTPNRO4GPZh0V7VRbiWwkRg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/w3c-xmlserializer": {
|
||||
"version": "5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-5.0.0.tgz",
|
||||
"integrity": "sha512-o8qghlI8NZHU1lLPrpi2+Uq7abh4GGPpYANlalzWxyWteJOCsr/P+oPBA49TOLu5FTZO4d3F9MnWJfiMo4BkmA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"xml-name-validator": "^5.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/webidl-conversions": {
|
||||
"version": "8.0.0",
|
||||
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-8.0.0.tgz",
|
||||
"integrity": "sha512-n4W4YFyz5JzOfQeA8oN7dUYpR+MBP3PIUsn2jLjWXwK5ASUzt0Jc/A5sAUZoCYFJRGF0FBKJ+1JjN43rNdsQzA==",
|
||||
"license": "BSD-2-Clause",
|
||||
"engines": {
|
||||
"node": ">=20"
|
||||
}
|
||||
},
|
||||
"node_modules/whatwg-encoding": {
|
||||
"version": "3.1.1",
|
||||
"resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-3.1.1.tgz",
|
||||
"integrity": "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"iconv-lite": "0.6.3"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/whatwg-mimetype": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-4.0.0.tgz",
|
||||
"integrity": "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/whatwg-url": {
|
||||
"version": "15.1.0",
|
||||
"resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-15.1.0.tgz",
|
||||
"integrity": "sha512-2ytDk0kiEj/yu90JOAp44PVPUkO9+jVhyf+SybKlRHSDlvOOZhdPIrr7xTH64l4WixO2cP+wQIcgujkGBPPz6g==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"tr46": "^6.0.0",
|
||||
"webidl-conversions": "^8.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20"
|
||||
}
|
||||
},
|
||||
"node_modules/ws": {
|
||||
"version": "8.18.3",
|
||||
"resolved": "https://registry.npmjs.org/ws/-/ws-8.18.3.tgz",
|
||||
"integrity": "sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=10.0.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"bufferutil": "^4.0.1",
|
||||
"utf-8-validate": ">=5.0.2"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"bufferutil": {
|
||||
"optional": true
|
||||
},
|
||||
"utf-8-validate": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/xml-name-validator": {
|
||||
"version": "5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-5.0.0.tgz",
|
||||
"integrity": "sha512-EvGK8EJ3DhaHfbRlETOWAS5pO9MZITeauHKJyb8wyajUfQUenkIg2MvLDTZ4T/TgIcm3HU0TFBgWWboAZ30UHg==",
|
||||
"license": "Apache-2.0",
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/xmlchars": {
|
||||
"version": "2.2.0",
|
||||
"resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz",
|
||||
"integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==",
|
||||
"license": "MIT"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"name": "brave-search",
|
||||
"version": "1.0.0",
|
||||
"type": "module",
|
||||
"description": "Headless web search via Brave Search - no browser required",
|
||||
"author": "Mario Zechner",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@mozilla/readability": "^0.6.0",
|
||||
"jsdom": "^27.0.1",
|
||||
"turndown": "^7.2.2",
|
||||
"turndown-plugin-gfm": "^1.0.2"
|
||||
}
|
||||
}
|
||||
+199
@@ -0,0 +1,199 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { Readability } from "@mozilla/readability";
|
||||
import { JSDOM } from "jsdom";
|
||||
import TurndownService from "turndown";
|
||||
import { gfm } from "turndown-plugin-gfm";
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
|
||||
const contentIndex = args.indexOf("--content");
|
||||
const fetchContent = contentIndex !== -1;
|
||||
if (fetchContent) args.splice(contentIndex, 1);
|
||||
|
||||
let numResults = 5;
|
||||
const nIndex = args.indexOf("-n");
|
||||
if (nIndex !== -1 && args[nIndex + 1]) {
|
||||
numResults = parseInt(args[nIndex + 1], 10);
|
||||
args.splice(nIndex, 2);
|
||||
}
|
||||
|
||||
// Parse country option
|
||||
let country = "US";
|
||||
const countryIndex = args.indexOf("--country");
|
||||
if (countryIndex !== -1 && args[countryIndex + 1]) {
|
||||
country = args[countryIndex + 1].toUpperCase();
|
||||
args.splice(countryIndex, 2);
|
||||
}
|
||||
|
||||
// Parse freshness option
|
||||
let freshness = null;
|
||||
const freshnessIndex = args.indexOf("--freshness");
|
||||
if (freshnessIndex !== -1 && args[freshnessIndex + 1]) {
|
||||
freshness = args[freshnessIndex + 1];
|
||||
args.splice(freshnessIndex, 2);
|
||||
}
|
||||
|
||||
const query = args.join(" ");
|
||||
|
||||
if (!query) {
|
||||
console.log("Usage: search.js <query> [-n <num>] [--content] [--country <code>] [--freshness <period>]");
|
||||
console.log("\nOptions:");
|
||||
console.log(" -n <num> Number of results (default: 5, max: 20)");
|
||||
console.log(" --content Fetch readable content as markdown");
|
||||
console.log(" --country <code> Country code for results (default: US)");
|
||||
console.log(" --freshness <period> Filter by time: pd (day), pw (week), pm (month), py (year)");
|
||||
console.log("\nEnvironment:");
|
||||
console.log(" BRAVE_API_KEY Required. Your Brave Search API key.");
|
||||
console.log("\nExamples:");
|
||||
console.log(' search.js "javascript async await"');
|
||||
console.log(' search.js "rust programming" -n 10');
|
||||
console.log(' search.js "climate change" --content');
|
||||
console.log(' search.js "news today" --freshness pd');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const apiKey = process.env.BRAVE_API_KEY;
|
||||
if (!apiKey) {
|
||||
console.error("Error: BRAVE_API_KEY environment variable is required.");
|
||||
console.error("Get your API key at: https://api-dashboard.search.brave.com/app/keys");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
async function fetchBraveResults(query, numResults, country, freshness) {
|
||||
const params = new URLSearchParams({
|
||||
q: query,
|
||||
count: Math.min(numResults, 20).toString(),
|
||||
country: country,
|
||||
});
|
||||
|
||||
if (freshness) {
|
||||
params.append("freshness", freshness);
|
||||
}
|
||||
|
||||
const url = `https://api.search.brave.com/res/v1/web/search?${params.toString()}`;
|
||||
|
||||
const response = await fetch(url, {
|
||||
headers: {
|
||||
"Accept": "application/json",
|
||||
"Accept-Encoding": "gzip",
|
||||
"X-Subscription-Token": apiKey,
|
||||
}
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
throw new Error(`HTTP ${response.status}: ${response.statusText}\n${errorText}`);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
|
||||
const results = [];
|
||||
|
||||
// Extract web results
|
||||
if (data.web && data.web.results) {
|
||||
for (const result of data.web.results) {
|
||||
if (results.length >= numResults) break;
|
||||
|
||||
results.push({
|
||||
title: result.title || "",
|
||||
link: result.url || "",
|
||||
snippet: result.description || "",
|
||||
age: result.age || result.page_age || "",
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
function htmlToMarkdown(html) {
|
||||
const turndown = new TurndownService({ headingStyle: "atx", codeBlockStyle: "fenced" });
|
||||
turndown.use(gfm);
|
||||
turndown.addRule("removeEmptyLinks", {
|
||||
filter: (node) => node.nodeName === "A" && !node.textContent?.trim(),
|
||||
replacement: () => "",
|
||||
});
|
||||
return turndown
|
||||
.turndown(html)
|
||||
.replace(/\[\\?\[\s*\\?\]\]\([^)]*\)/g, "")
|
||||
.replace(/ +/g, " ")
|
||||
.replace(/\s+,/g, ",")
|
||||
.replace(/\s+\./g, ".")
|
||||
.replace(/\n{3,}/g, "\n\n")
|
||||
.trim();
|
||||
}
|
||||
|
||||
async function fetchPageContent(url) {
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
headers: {
|
||||
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36",
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
},
|
||||
signal: AbortSignal.timeout(10000),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
return `(HTTP ${response.status})`;
|
||||
}
|
||||
|
||||
const html = await response.text();
|
||||
const dom = new JSDOM(html, { url });
|
||||
const reader = new Readability(dom.window.document);
|
||||
const article = reader.parse();
|
||||
|
||||
if (article && article.content) {
|
||||
return htmlToMarkdown(article.content).substring(0, 5000);
|
||||
}
|
||||
|
||||
// Fallback: try to get main content
|
||||
const fallbackDoc = new JSDOM(html, { url });
|
||||
const body = fallbackDoc.window.document;
|
||||
body.querySelectorAll("script, style, noscript, nav, header, footer, aside").forEach(el => el.remove());
|
||||
const main = body.querySelector("main, article, [role='main'], .content, #content") || body.body;
|
||||
const text = main?.textContent || "";
|
||||
|
||||
if (text.trim().length > 100) {
|
||||
return text.trim().substring(0, 5000);
|
||||
}
|
||||
|
||||
return "(Could not extract content)";
|
||||
} catch (e) {
|
||||
return `(Error: ${e.message})`;
|
||||
}
|
||||
}
|
||||
|
||||
// Main
|
||||
try {
|
||||
const results = await fetchBraveResults(query, numResults, country, freshness);
|
||||
|
||||
if (results.length === 0) {
|
||||
console.error("No results found.");
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
if (fetchContent) {
|
||||
for (const result of results) {
|
||||
result.content = await fetchPageContent(result.link);
|
||||
}
|
||||
}
|
||||
|
||||
for (let i = 0; i < results.length; i++) {
|
||||
const r = results[i];
|
||||
console.log(`--- Result ${i + 1} ---`);
|
||||
console.log(`Title: ${r.title}`);
|
||||
console.log(`Link: ${r.link}`);
|
||||
if (r.age) {
|
||||
console.log(`Age: ${r.age}`);
|
||||
}
|
||||
console.log(`Snippet: ${r.snippet}`);
|
||||
if (r.content) {
|
||||
console.log(`Content:\n${r.content}`);
|
||||
}
|
||||
console.log("");
|
||||
}
|
||||
} catch (e) {
|
||||
console.error(`Error: ${e.message}`);
|
||||
process.exit(1);
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
# Headless Chrome profile (copy of user's Chrome profile)
|
||||
.headless-profile/
|
||||
|
||||
# Node modules
|
||||
node_modules/
|
||||
|
||||
# Debug files
|
||||
debug-*.png
|
||||
@@ -0,0 +1,196 @@
|
||||
---
|
||||
name: browser-tools
|
||||
description: Interactive browser automation via Chrome DevTools Protocol. Use when you need to interact with web pages, test frontends, or when user interaction with a visible browser is required.
|
||||
---
|
||||
|
||||
# Browser Tools
|
||||
|
||||
Chrome DevTools Protocol tools for agent-assisted web automation. These tools connect to Chrome running on `:9222` with remote debugging enabled.
|
||||
|
||||
## Setup
|
||||
|
||||
Run once before first use:
|
||||
|
||||
```bash
|
||||
cd {baseDir}/browser-tools
|
||||
npm install
|
||||
```
|
||||
|
||||
## Start Chrome
|
||||
|
||||
```bash
|
||||
{baseDir}/browser-start.js # Fresh profile
|
||||
{baseDir}/browser-start.js --profile # Copy user's profile (cookies, logins)
|
||||
```
|
||||
|
||||
Launch Chrome with remote debugging on `:9222`. Use `--profile` to preserve user's authentication state.
|
||||
|
||||
## Navigate
|
||||
|
||||
```bash
|
||||
{baseDir}/browser-nav.js https://example.com
|
||||
{baseDir}/browser-nav.js https://example.com --new
|
||||
```
|
||||
|
||||
Navigate to URLs. Use `--new` flag to open in a new tab instead of reusing current tab.
|
||||
|
||||
## Evaluate JavaScript
|
||||
|
||||
```bash
|
||||
{baseDir}/browser-eval.js 'document.title'
|
||||
{baseDir}/browser-eval.js 'document.querySelectorAll("a").length'
|
||||
```
|
||||
|
||||
Execute JavaScript in the active tab. Code runs in async context. Use this to extract data, inspect page state, or perform DOM operations programmatically.
|
||||
|
||||
## Screenshot
|
||||
|
||||
```bash
|
||||
{baseDir}/browser-screenshot.js
|
||||
```
|
||||
|
||||
Capture current viewport and return temporary file path. Use this to visually inspect page state or verify UI changes.
|
||||
|
||||
## Pick Elements
|
||||
|
||||
```bash
|
||||
{baseDir}/browser-pick.js "Click the submit button"
|
||||
```
|
||||
|
||||
**IMPORTANT**: Use this tool when the user wants to select specific DOM elements on the page. This launches an interactive picker that lets the user click elements to select them. The user can select multiple elements (Cmd/Ctrl+Click) and press Enter when done. The tool returns CSS selectors for the selected elements.
|
||||
|
||||
Common use cases:
|
||||
- User says "I want to click that button" → Use this tool to let them select it
|
||||
- User says "extract data from these items" → Use this tool to let them select the elements
|
||||
- When you need specific selectors but the page structure is complex or ambiguous
|
||||
|
||||
## Cookies
|
||||
|
||||
```bash
|
||||
{baseDir}/browser-cookies.js
|
||||
```
|
||||
|
||||
Display all cookies for the current tab including domain, path, httpOnly, and secure flags. Use this to debug authentication issues or inspect session state.
|
||||
|
||||
## Extract Page Content
|
||||
|
||||
```bash
|
||||
{baseDir}/browser-content.js https://example.com
|
||||
```
|
||||
|
||||
Navigate to a URL and extract readable content as markdown. Uses Mozilla Readability for article extraction and Turndown for HTML-to-markdown conversion. Works on pages with JavaScript content (waits for page to load).
|
||||
|
||||
## When to Use
|
||||
|
||||
- Testing frontend code in a real browser
|
||||
- Interacting with pages that require JavaScript
|
||||
- When user needs to visually see or interact with a page
|
||||
- Debugging authentication or session issues
|
||||
- Scraping dynamic content that requires JS execution
|
||||
|
||||
---
|
||||
|
||||
## Efficiency Guide
|
||||
|
||||
### DOM Inspection Over Screenshots
|
||||
|
||||
**Don't** take screenshots to see page state. **Do** parse the DOM directly:
|
||||
|
||||
```javascript
|
||||
// Get page structure
|
||||
document.body.innerHTML.slice(0, 5000)
|
||||
|
||||
// Find interactive elements
|
||||
Array.from(document.querySelectorAll('button, input, [role="button"]')).map(e => ({
|
||||
id: e.id,
|
||||
text: e.textContent.trim(),
|
||||
class: e.className
|
||||
}))
|
||||
```
|
||||
|
||||
### Complex Scripts in Single Calls
|
||||
|
||||
Wrap everything in an IIFE to run multi-statement code:
|
||||
|
||||
```javascript
|
||||
(function() {
|
||||
// Multiple operations
|
||||
const data = document.querySelector('#target').textContent;
|
||||
const buttons = document.querySelectorAll('button');
|
||||
|
||||
// Interactions
|
||||
buttons[0].click();
|
||||
|
||||
// Return results
|
||||
return JSON.stringify({ data, buttonCount: buttons.length });
|
||||
})()
|
||||
```
|
||||
|
||||
### Batch Interactions
|
||||
|
||||
**Don't** make separate calls for each click. **Do** batch them:
|
||||
|
||||
```javascript
|
||||
(function() {
|
||||
const actions = ["btn1", "btn2", "btn3"];
|
||||
actions.forEach(id => document.getElementById(id).click());
|
||||
return "Done";
|
||||
})()
|
||||
```
|
||||
|
||||
### Typing/Input Sequences
|
||||
|
||||
```javascript
|
||||
(function() {
|
||||
const text = "HELLO";
|
||||
for (const char of text) {
|
||||
document.getElementById("key-" + char).click();
|
||||
}
|
||||
document.getElementById("submit").click();
|
||||
return "Submitted: " + text;
|
||||
})()
|
||||
```
|
||||
|
||||
### Reading App/Game State
|
||||
|
||||
Extract structured state in one call:
|
||||
|
||||
```javascript
|
||||
(function() {
|
||||
const state = {
|
||||
score: document.querySelector('.score')?.textContent,
|
||||
status: document.querySelector('.status')?.className,
|
||||
items: Array.from(document.querySelectorAll('.item')).map(el => ({
|
||||
text: el.textContent,
|
||||
active: el.classList.contains('active')
|
||||
}))
|
||||
};
|
||||
return JSON.stringify(state, null, 2);
|
||||
})()
|
||||
```
|
||||
|
||||
### Waiting for Updates
|
||||
|
||||
If DOM updates after actions, add a small delay with bash:
|
||||
|
||||
```bash
|
||||
sleep 0.5 && {baseDir}/browser-eval.js '...'
|
||||
```
|
||||
|
||||
### Investigate Before Interacting
|
||||
|
||||
Always start by understanding the page structure:
|
||||
|
||||
```javascript
|
||||
(function() {
|
||||
return {
|
||||
title: document.title,
|
||||
forms: document.forms.length,
|
||||
buttons: document.querySelectorAll('button').length,
|
||||
inputs: document.querySelectorAll('input').length,
|
||||
mainContent: document.body.innerHTML.slice(0, 3000)
|
||||
};
|
||||
})()
|
||||
```
|
||||
|
||||
Then target specific elements based on what you find.
|
||||
@@ -0,0 +1,103 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import puppeteer from "puppeteer-core";
|
||||
import { Readability } from "@mozilla/readability";
|
||||
import { JSDOM } from "jsdom";
|
||||
import TurndownService from "turndown";
|
||||
import { gfm } from "turndown-plugin-gfm";
|
||||
|
||||
// Global timeout - exit if script takes too long
|
||||
const TIMEOUT = 30000;
|
||||
const timeoutId = setTimeout(() => {
|
||||
console.error("✗ Timeout after 30s");
|
||||
process.exit(1);
|
||||
}, TIMEOUT).unref();
|
||||
|
||||
const url = process.argv[2];
|
||||
|
||||
if (!url) {
|
||||
console.log("Usage: browser-content.js <url>");
|
||||
console.log("\nExtracts readable content from a URL as markdown.");
|
||||
console.log("\nExamples:");
|
||||
console.log(" browser-content.js https://example.com");
|
||||
console.log(" browser-content.js https://en.wikipedia.org/wiki/Rust_(programming_language)");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const b = await Promise.race([
|
||||
puppeteer.connect({
|
||||
browserURL: "http://localhost:9222",
|
||||
defaultViewport: null,
|
||||
}),
|
||||
new Promise((_, reject) => setTimeout(() => reject(new Error("timeout")), 5000)),
|
||||
]).catch((e) => {
|
||||
console.error("✗ Could not connect to browser:", e.message);
|
||||
console.error(" Run: browser-start.js");
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
const p = (await b.pages()).at(-1);
|
||||
if (!p) {
|
||||
console.error("✗ No active tab found");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
await Promise.race([
|
||||
p.goto(url, { waitUntil: "networkidle2" }),
|
||||
new Promise((r) => setTimeout(r, 10000)),
|
||||
]).catch(() => {});
|
||||
|
||||
// Get HTML via CDP (works even with TrustedScriptURL restrictions)
|
||||
const client = await p.createCDPSession();
|
||||
const { root } = await client.send("DOM.getDocument", { depth: -1, pierce: true });
|
||||
const { outerHTML } = await client.send("DOM.getOuterHTML", { nodeId: root.nodeId });
|
||||
await client.detach();
|
||||
|
||||
const finalUrl = p.url();
|
||||
|
||||
// Extract with Readability
|
||||
const doc = new JSDOM(outerHTML, { url: finalUrl });
|
||||
const reader = new Readability(doc.window.document);
|
||||
const article = reader.parse();
|
||||
|
||||
// Convert to markdown
|
||||
function htmlToMarkdown(html) {
|
||||
const turndown = new TurndownService({ headingStyle: "atx", codeBlockStyle: "fenced" });
|
||||
turndown.use(gfm);
|
||||
turndown.addRule("removeEmptyLinks", {
|
||||
filter: (node) => node.nodeName === "A" && !node.textContent?.trim(),
|
||||
replacement: () => "",
|
||||
});
|
||||
return turndown
|
||||
.turndown(html)
|
||||
.replace(/\[\\?\[\s*\\?\]\]\([^)]*\)/g, "")
|
||||
.replace(/ +/g, " ")
|
||||
.replace(/\s+,/g, ",")
|
||||
.replace(/\s+\./g, ".")
|
||||
.replace(/\n{3,}/g, "\n\n")
|
||||
.trim();
|
||||
}
|
||||
|
||||
let content;
|
||||
if (article && article.content) {
|
||||
content = htmlToMarkdown(article.content);
|
||||
} else {
|
||||
// Fallback
|
||||
const fallbackDoc = new JSDOM(outerHTML, { url: finalUrl });
|
||||
const fallbackBody = fallbackDoc.window.document;
|
||||
fallbackBody.querySelectorAll("script, style, noscript, nav, header, footer, aside").forEach((el) => el.remove());
|
||||
const main = fallbackBody.querySelector("main, article, [role='main'], .content, #content") || fallbackBody.body;
|
||||
const fallbackHtml = main?.innerHTML || "";
|
||||
if (fallbackHtml.trim().length > 100) {
|
||||
content = htmlToMarkdown(fallbackHtml);
|
||||
} else {
|
||||
content = "(Could not extract content)";
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`URL: ${finalUrl}`);
|
||||
if (article?.title) console.log(`Title: ${article.title}`);
|
||||
console.log("");
|
||||
console.log(content);
|
||||
|
||||
process.exit(0);
|
||||
@@ -0,0 +1,35 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import puppeteer from "puppeteer-core";
|
||||
|
||||
const b = await Promise.race([
|
||||
puppeteer.connect({
|
||||
browserURL: "http://localhost:9222",
|
||||
defaultViewport: null,
|
||||
}),
|
||||
new Promise((_, reject) => setTimeout(() => reject(new Error("timeout")), 5000)),
|
||||
]).catch((e) => {
|
||||
console.error("✗ Could not connect to browser:", e.message);
|
||||
console.error(" Run: browser-start.js");
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
const p = (await b.pages()).at(-1);
|
||||
|
||||
if (!p) {
|
||||
console.error("✗ No active tab found");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const cookies = await p.cookies();
|
||||
|
||||
for (const cookie of cookies) {
|
||||
console.log(`${cookie.name}: ${cookie.value}`);
|
||||
console.log(` domain: ${cookie.domain}`);
|
||||
console.log(` path: ${cookie.path}`);
|
||||
console.log(` httpOnly: ${cookie.httpOnly}`);
|
||||
console.log(` secure: ${cookie.secure}`);
|
||||
console.log("");
|
||||
}
|
||||
|
||||
await b.disconnect();
|
||||
@@ -0,0 +1,53 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import puppeteer from "puppeteer-core";
|
||||
|
||||
const code = process.argv.slice(2).join(" ");
|
||||
if (!code) {
|
||||
console.log("Usage: browser-eval.js 'code'");
|
||||
console.log("\nExamples:");
|
||||
console.log(' browser-eval.js "document.title"');
|
||||
console.log(' browser-eval.js "document.querySelectorAll(\'a\').length"');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const b = await Promise.race([
|
||||
puppeteer.connect({
|
||||
browserURL: "http://localhost:9222",
|
||||
defaultViewport: null,
|
||||
}),
|
||||
new Promise((_, reject) => setTimeout(() => reject(new Error("timeout")), 5000)),
|
||||
]).catch((e) => {
|
||||
console.error("✗ Could not connect to browser:", e.message);
|
||||
console.error(" Run: browser-start.js");
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
const p = (await b.pages()).at(-1);
|
||||
|
||||
if (!p) {
|
||||
console.error("✗ No active tab found");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const result = await p.evaluate((c) => {
|
||||
const AsyncFunction = (async () => {}).constructor;
|
||||
return new AsyncFunction(`return (${c})`)();
|
||||
}, code);
|
||||
|
||||
if (Array.isArray(result)) {
|
||||
for (let i = 0; i < result.length; i++) {
|
||||
if (i > 0) console.log("");
|
||||
for (const [key, value] of Object.entries(result[i])) {
|
||||
console.log(`${key}: ${value}`);
|
||||
}
|
||||
}
|
||||
} else if (typeof result === "object" && result !== null) {
|
||||
for (const [key, value] of Object.entries(result)) {
|
||||
console.log(`${key}: ${value}`);
|
||||
}
|
||||
} else {
|
||||
console.log(result);
|
||||
}
|
||||
|
||||
await b.disconnect();
|
||||
@@ -0,0 +1,108 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
/**
|
||||
* Hacker News Scraper
|
||||
*
|
||||
* Fetches and parses submissions from Hacker News front page.
|
||||
* Usage: node browser-hn-scraper.js [--limit <number>]
|
||||
*/
|
||||
|
||||
import * as cheerio from 'cheerio';
|
||||
|
||||
/**
|
||||
* Scrapes Hacker News front page
|
||||
* @param {number} limit - Maximum number of submissions to return (default: 30)
|
||||
* @returns {Promise<Array>} Array of submission objects
|
||||
*/
|
||||
async function scrapeHackerNews(limit = 30) {
|
||||
const url = 'https://news.ycombinator.com';
|
||||
|
||||
try {
|
||||
const response = await fetch(url);
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP error! status: ${response.status}`);
|
||||
}
|
||||
|
||||
const html = await response.text();
|
||||
const $ = cheerio.load(html);
|
||||
const submissions = [];
|
||||
|
||||
// Each submission has class 'athing'
|
||||
$('.athing').each((index, element) => {
|
||||
if (submissions.length >= limit) return false; // Stop when limit reached
|
||||
|
||||
const $element = $(element);
|
||||
const id = $element.attr('id');
|
||||
|
||||
// Get title and URL from titleline
|
||||
const $titleLine = $element.find('.titleline > a').first();
|
||||
const title = $titleLine.text().trim();
|
||||
const url = $titleLine.attr('href');
|
||||
|
||||
// Get the next row which contains metadata (points, author, comments)
|
||||
const $metadataRow = $element.next();
|
||||
const $subtext = $metadataRow.find('.subtext');
|
||||
|
||||
// Get points
|
||||
const $score = $subtext.find(`#score_${id}`);
|
||||
const pointsText = $score.text();
|
||||
const points = pointsText ? parseInt(pointsText.match(/\d+/)?.[0] || '0') : 0;
|
||||
|
||||
// Get author
|
||||
const author = $subtext.find('.hnuser').text().trim();
|
||||
|
||||
// Get time
|
||||
const time = $subtext.find('.age').attr('title') || $subtext.find('.age').text().trim();
|
||||
|
||||
// Get comments count
|
||||
const $commentsLink = $subtext.find('a').last();
|
||||
const commentsText = $commentsLink.text();
|
||||
let commentsCount = 0;
|
||||
|
||||
if (commentsText.includes('comment')) {
|
||||
const match = commentsText.match(/(\d+)/);
|
||||
commentsCount = match ? parseInt(match[0]) : 0;
|
||||
}
|
||||
|
||||
submissions.push({
|
||||
id,
|
||||
title,
|
||||
url,
|
||||
points,
|
||||
author,
|
||||
time,
|
||||
comments: commentsCount,
|
||||
hnUrl: `https://news.ycombinator.com/item?id=${id}`
|
||||
});
|
||||
});
|
||||
|
||||
return submissions;
|
||||
} catch (error) {
|
||||
console.error('Error scraping Hacker News:', error.message);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// CLI interface
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
const args = process.argv.slice(2);
|
||||
let limit = 30;
|
||||
|
||||
// Parse --limit argument
|
||||
const limitIndex = args.indexOf('--limit');
|
||||
if (limitIndex !== -1 && args[limitIndex + 1]) {
|
||||
limit = parseInt(args[limitIndex + 1]);
|
||||
}
|
||||
|
||||
scrapeHackerNews(limit)
|
||||
.then(submissions => {
|
||||
console.log(JSON.stringify(submissions, null, 2));
|
||||
console.error(`\n✓ Scraped ${submissions.length} submissions`);
|
||||
})
|
||||
.catch(error => {
|
||||
console.error('Failed to scrape:', error.message);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
|
||||
export { scrapeHackerNews };
|
||||
@@ -0,0 +1,44 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import puppeteer from "puppeteer-core";
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const newTab = args.includes("--new");
|
||||
const reload = args.includes("--reload");
|
||||
const url = args.find(a => !a.startsWith("--"));
|
||||
|
||||
if (!url) {
|
||||
console.log("Usage: browser-nav.js <url> [--new] [--reload]");
|
||||
console.log("\nExamples:");
|
||||
console.log(" browser-nav.js https://example.com # Navigate current tab");
|
||||
console.log(" browser-nav.js https://example.com --new # Open in new tab");
|
||||
console.log(" browser-nav.js https://example.com --reload # Navigate and force reload");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const b = await Promise.race([
|
||||
puppeteer.connect({
|
||||
browserURL: "http://localhost:9222",
|
||||
defaultViewport: null,
|
||||
}),
|
||||
new Promise((_, reject) => setTimeout(() => reject(new Error("timeout")), 5000)),
|
||||
]).catch((e) => {
|
||||
console.error("✗ Could not connect to browser:", e.message);
|
||||
console.error(" Run: browser-start.js");
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
if (newTab) {
|
||||
const p = await b.newPage();
|
||||
await p.goto(url, { waitUntil: "domcontentloaded" });
|
||||
console.log("✓ Opened:", url);
|
||||
} else {
|
||||
const p = (await b.pages()).at(-1);
|
||||
await p.goto(url, { waitUntil: "domcontentloaded" });
|
||||
if (reload) {
|
||||
await p.reload({ waitUntil: "domcontentloaded" });
|
||||
}
|
||||
console.log("✓ Navigated to:", url);
|
||||
}
|
||||
|
||||
await b.disconnect();
|
||||
@@ -0,0 +1,162 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import puppeteer from "puppeteer-core";
|
||||
|
||||
const message = process.argv.slice(2).join(" ");
|
||||
if (!message) {
|
||||
console.log("Usage: browser-pick.js 'message'");
|
||||
console.log("\nExample:");
|
||||
console.log(' browser-pick.js "Click the submit button"');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const b = await Promise.race([
|
||||
puppeteer.connect({
|
||||
browserURL: "http://localhost:9222",
|
||||
defaultViewport: null,
|
||||
}),
|
||||
new Promise((_, reject) => setTimeout(() => reject(new Error("timeout")), 5000)),
|
||||
]).catch((e) => {
|
||||
console.error("✗ Could not connect to browser:", e.message);
|
||||
console.error(" Run: browser-start.js");
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
const p = (await b.pages()).at(-1);
|
||||
|
||||
if (!p) {
|
||||
console.error("✗ No active tab found");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Inject pick() helper into current page
|
||||
await p.evaluate(() => {
|
||||
if (!window.pick) {
|
||||
window.pick = async (message) => {
|
||||
if (!message) {
|
||||
throw new Error("pick() requires a message parameter");
|
||||
}
|
||||
return new Promise((resolve) => {
|
||||
const selections = [];
|
||||
const selectedElements = new Set();
|
||||
|
||||
const overlay = document.createElement("div");
|
||||
overlay.style.cssText =
|
||||
"position:fixed;top:0;left:0;width:100%;height:100%;z-index:2147483647;pointer-events:none";
|
||||
|
||||
const highlight = document.createElement("div");
|
||||
highlight.style.cssText =
|
||||
"position:absolute;border:2px solid #3b82f6;background:rgba(59,130,246,0.1);transition:all 0.1s";
|
||||
overlay.appendChild(highlight);
|
||||
|
||||
const banner = document.createElement("div");
|
||||
banner.style.cssText =
|
||||
"position:fixed;bottom:20px;left:50%;transform:translateX(-50%);background:#1f2937;color:white;padding:12px 24px;border-radius:8px;font:14px sans-serif;box-shadow:0 4px 12px rgba(0,0,0,0.3);pointer-events:auto;z-index:2147483647";
|
||||
|
||||
const updateBanner = () => {
|
||||
banner.textContent = `${message} (${selections.length} selected, Cmd/Ctrl+click to add, Enter to finish, ESC to cancel)`;
|
||||
};
|
||||
updateBanner();
|
||||
|
||||
document.body.append(banner, overlay);
|
||||
|
||||
const cleanup = () => {
|
||||
document.removeEventListener("mousemove", onMove, true);
|
||||
document.removeEventListener("click", onClick, true);
|
||||
document.removeEventListener("keydown", onKey, true);
|
||||
overlay.remove();
|
||||
banner.remove();
|
||||
selectedElements.forEach((el) => {
|
||||
el.style.outline = "";
|
||||
});
|
||||
};
|
||||
|
||||
const onMove = (e) => {
|
||||
const el = document.elementFromPoint(e.clientX, e.clientY);
|
||||
if (!el || overlay.contains(el) || banner.contains(el)) return;
|
||||
const r = el.getBoundingClientRect();
|
||||
highlight.style.cssText = `position:absolute;border:2px solid #3b82f6;background:rgba(59,130,246,0.1);top:${r.top}px;left:${r.left}px;width:${r.width}px;height:${r.height}px`;
|
||||
};
|
||||
|
||||
const buildElementInfo = (el) => {
|
||||
const parents = [];
|
||||
let current = el.parentElement;
|
||||
while (current && current !== document.body) {
|
||||
const parentInfo = current.tagName.toLowerCase();
|
||||
const id = current.id ? `#${current.id}` : "";
|
||||
const cls = current.className
|
||||
? `.${current.className.trim().split(/\s+/).join(".")}`
|
||||
: "";
|
||||
parents.push(parentInfo + id + cls);
|
||||
current = current.parentElement;
|
||||
}
|
||||
|
||||
return {
|
||||
tag: el.tagName.toLowerCase(),
|
||||
id: el.id || null,
|
||||
class: el.className || null,
|
||||
text: el.textContent?.trim().slice(0, 200) || null,
|
||||
html: el.outerHTML.slice(0, 500),
|
||||
parents: parents.join(" > "),
|
||||
};
|
||||
};
|
||||
|
||||
const onClick = (e) => {
|
||||
if (banner.contains(e.target)) return;
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
const el = document.elementFromPoint(e.clientX, e.clientY);
|
||||
if (!el || overlay.contains(el) || banner.contains(el)) return;
|
||||
|
||||
if (e.metaKey || e.ctrlKey) {
|
||||
if (!selectedElements.has(el)) {
|
||||
selectedElements.add(el);
|
||||
el.style.outline = "3px solid #10b981";
|
||||
selections.push(buildElementInfo(el));
|
||||
updateBanner();
|
||||
}
|
||||
} else {
|
||||
cleanup();
|
||||
const info = buildElementInfo(el);
|
||||
resolve(selections.length > 0 ? selections : info);
|
||||
}
|
||||
};
|
||||
|
||||
const onKey = (e) => {
|
||||
if (e.key === "Escape") {
|
||||
e.preventDefault();
|
||||
cleanup();
|
||||
resolve(null);
|
||||
} else if (e.key === "Enter" && selections.length > 0) {
|
||||
e.preventDefault();
|
||||
cleanup();
|
||||
resolve(selections);
|
||||
}
|
||||
};
|
||||
|
||||
document.addEventListener("mousemove", onMove, true);
|
||||
document.addEventListener("click", onClick, true);
|
||||
document.addEventListener("keydown", onKey, true);
|
||||
});
|
||||
};
|
||||
}
|
||||
});
|
||||
|
||||
const result = await p.evaluate((msg) => window.pick(msg), message);
|
||||
|
||||
if (Array.isArray(result)) {
|
||||
for (let i = 0; i < result.length; i++) {
|
||||
if (i > 0) console.log("");
|
||||
for (const [key, value] of Object.entries(result[i])) {
|
||||
console.log(`${key}: ${value}`);
|
||||
}
|
||||
}
|
||||
} else if (typeof result === "object" && result !== null) {
|
||||
for (const [key, value] of Object.entries(result)) {
|
||||
console.log(`${key}: ${value}`);
|
||||
}
|
||||
} else {
|
||||
console.log(result);
|
||||
}
|
||||
|
||||
await b.disconnect();
|
||||
@@ -0,0 +1,34 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import puppeteer from "puppeteer-core";
|
||||
|
||||
const b = await Promise.race([
|
||||
puppeteer.connect({
|
||||
browserURL: "http://localhost:9222",
|
||||
defaultViewport: null,
|
||||
}),
|
||||
new Promise((_, reject) => setTimeout(() => reject(new Error("timeout")), 5000)),
|
||||
]).catch((e) => {
|
||||
console.error("✗ Could not connect to browser:", e.message);
|
||||
console.error(" Run: browser-start.js");
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
const p = (await b.pages()).at(-1);
|
||||
|
||||
if (!p) {
|
||||
console.error("✗ No active tab found");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
|
||||
const filename = `screenshot-${timestamp}.png`;
|
||||
const filepath = join(tmpdir(), filename);
|
||||
|
||||
await p.screenshot({ path: filepath });
|
||||
|
||||
console.log(filepath);
|
||||
|
||||
await b.disconnect();
|
||||
@@ -0,0 +1,86 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { spawn, execSync } from "node:child_process";
|
||||
import puppeteer from "puppeteer-core";
|
||||
|
||||
const useProfile = process.argv[2] === "--profile";
|
||||
|
||||
if (process.argv[2] && process.argv[2] !== "--profile") {
|
||||
console.log("Usage: browser-start.js [--profile]");
|
||||
console.log("\nOptions:");
|
||||
console.log(" --profile Copy your default Chrome profile (cookies, logins)");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const SCRAPING_DIR = `${process.env.HOME}/.cache/browser-tools`;
|
||||
|
||||
// Check if already running on :9222
|
||||
try {
|
||||
const browser = await puppeteer.connect({
|
||||
browserURL: "http://localhost:9222",
|
||||
defaultViewport: null,
|
||||
});
|
||||
await browser.disconnect();
|
||||
console.log("✓ Chrome already running on :9222");
|
||||
process.exit(0);
|
||||
} catch {}
|
||||
|
||||
// Setup profile directory
|
||||
execSync(`mkdir -p "${SCRAPING_DIR}"`, { stdio: "ignore" });
|
||||
|
||||
// Remove SingletonLock to allow new instance
|
||||
try {
|
||||
execSync(`rm -f "${SCRAPING_DIR}/SingletonLock" "${SCRAPING_DIR}/SingletonSocket" "${SCRAPING_DIR}/SingletonCookie"`, { stdio: "ignore" });
|
||||
} catch {}
|
||||
|
||||
if (useProfile) {
|
||||
console.log("Syncing profile...");
|
||||
execSync(
|
||||
`rsync -a --delete \
|
||||
--exclude='SingletonLock' \
|
||||
--exclude='SingletonSocket' \
|
||||
--exclude='SingletonCookie' \
|
||||
--exclude='*/Sessions/*' \
|
||||
--exclude='*/Current Session' \
|
||||
--exclude='*/Current Tabs' \
|
||||
--exclude='*/Last Session' \
|
||||
--exclude='*/Last Tabs' \
|
||||
"${process.env.HOME}/Library/Application Support/Google/Chrome/" "${SCRAPING_DIR}/"`,
|
||||
{ stdio: "pipe" },
|
||||
);
|
||||
}
|
||||
|
||||
// Start Chrome with flags to force new instance
|
||||
spawn(
|
||||
"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
|
||||
[
|
||||
"--remote-debugging-port=9222",
|
||||
`--user-data-dir=${SCRAPING_DIR}`,
|
||||
"--no-first-run",
|
||||
"--no-default-browser-check",
|
||||
],
|
||||
{ detached: true, stdio: "ignore" },
|
||||
).unref();
|
||||
|
||||
// Wait for Chrome to be ready
|
||||
let connected = false;
|
||||
for (let i = 0; i < 30; i++) {
|
||||
try {
|
||||
const browser = await puppeteer.connect({
|
||||
browserURL: "http://localhost:9222",
|
||||
defaultViewport: null,
|
||||
});
|
||||
await browser.disconnect();
|
||||
connected = true;
|
||||
break;
|
||||
} catch {
|
||||
await new Promise((r) => setTimeout(r, 500));
|
||||
}
|
||||
}
|
||||
|
||||
if (!connected) {
|
||||
console.error("✗ Failed to connect to Chrome");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log(`✓ Chrome started on :9222${useProfile ? " with your profile" : ""}`);
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"name": "browser-tools",
|
||||
"version": "1.0.0",
|
||||
"type": "module",
|
||||
"description": "Minimal CDP tools for collaborative site exploration",
|
||||
"author": "Mario Zechner",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@mozilla/readability": "^0.6.0",
|
||||
"cheerio": "^1.1.2",
|
||||
"jsdom": "^27.0.1",
|
||||
"puppeteer": "^24.31.0",
|
||||
"puppeteer-core": "^23.11.1",
|
||||
"puppeteer-extra": "^3.3.6",
|
||||
"puppeteer-extra-plugin-stealth": "^2.11.2",
|
||||
"turndown": "^7.2.2",
|
||||
"turndown-plugin-gfm": "^1.0.2"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
---
|
||||
name: youtube-transcript
|
||||
description: Fetch transcripts from YouTube videos for summarization and analysis.
|
||||
---
|
||||
|
||||
# YouTube Transcript
|
||||
|
||||
Fetch transcripts from YouTube videos.
|
||||
|
||||
## Setup
|
||||
|
||||
```bash
|
||||
cd {baseDir}
|
||||
npm install
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
```bash
|
||||
{baseDir}/transcript.js <video-id-or-url>
|
||||
```
|
||||
|
||||
Accepts video ID or full URL:
|
||||
- `EBw7gsDPAYQ`
|
||||
- `https://www.youtube.com/watch?v=EBw7gsDPAYQ`
|
||||
- `https://youtu.be/EBw7gsDPAYQ`
|
||||
|
||||
## Output
|
||||
|
||||
Timestamped transcript entries:
|
||||
|
||||
```
|
||||
[0:00] All right. So, I got this UniFi Theta
|
||||
[0:15] I took the camera out, painted it
|
||||
[1:23] And here's the final result
|
||||
```
|
||||
|
||||
## Notes
|
||||
|
||||
- Requires the video to have captions/transcripts available
|
||||
- Works with auto-generated and manual transcripts
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"name": "youtube-transcript",
|
||||
"version": "1.0.0",
|
||||
"type": "module",
|
||||
"dependencies": {
|
||||
"youtube-transcript-plus": "^1.0.4"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { YoutubeTranscript } from 'youtube-transcript-plus';
|
||||
|
||||
const videoId = process.argv[2];
|
||||
|
||||
if (!videoId) {
|
||||
console.error('Usage: transcript.js <video-id-or-url>');
|
||||
console.error('Example: transcript.js EBw7gsDPAYQ');
|
||||
console.error('Example: transcript.js https://www.youtube.com/watch?v=EBw7gsDPAYQ');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Extract video ID if full URL is provided
|
||||
let extractedId = videoId;
|
||||
if (videoId.includes('youtube.com') || videoId.includes('youtu.be')) {
|
||||
const match = videoId.match(/(?:v=|youtu\.be\/)([a-zA-Z0-9_-]{11})/);
|
||||
if (match) {
|
||||
extractedId = match[1];
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const transcript = await YoutubeTranscript.fetchTranscript(extractedId);
|
||||
|
||||
for (const entry of transcript) {
|
||||
const timestamp = formatTimestamp(entry.offset / 1000);
|
||||
console.log(`[${timestamp}] ${entry.text}`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error fetching transcript:', error.message);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
function formatTimestamp(seconds) {
|
||||
const h = Math.floor(seconds / 3600);
|
||||
const m = Math.floor((seconds % 3600) / 60);
|
||||
const s = Math.floor(seconds % 60);
|
||||
|
||||
if (h > 0) {
|
||||
return `${h}:${m.toString().padStart(2, '0')}:${s.toString().padStart(2, '0')}`;
|
||||
}
|
||||
return `${m}:${s.toString().padStart(2, '0')}`;
|
||||
}
|
||||
Reference in New Issue
Block a user