Public
HTTP (deprecated)
Val Town is a social website to write and deploy JavaScript.
Build APIs and schedule functions from your browser.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
import { PuppeteerDeno } from "https://deno.land/x/puppeteer@16.2.0/src/deno/Puppeteer.ts";
import { convertRelativeDateToString } from "https://esm.town/v/alexdphan/convertRelativeDateToString";
export interface Website {
source: string;
url: string;
title: string;
date_published: string;
}
export async function hackerNewsSearch({
query = "Artificial Intelligence",
pages = 3,
apiKey = Deno.env.get("BROWSERBASE_API_KEY"),
}: {
query?: string;
pages?: number;
apiKey: string;
}): Promise<Website[]> {
let browser;
try {
console.log("Initializing PuppeteerDeno...");
const puppeteer = new PuppeteerDeno({
productName: "chrome",
});
console.log("Connecting to BrowserBase...");
browser = await puppeteer.connect({
browserWSEndpoint: `wss://connect.browserbase.com?apiKey=${apiKey}`,
ignoreHTTPSErrors: true,
});
const page = await browser.newPage();
console.info("Navigating to Hacker News...");
await page.goto("https://hn.algolia.com/", { waitUntil: "networkidle0" });
// Set sorting to 'Date'
await page.waitForSelector(".SearchFilters_filterContainer .Dropdown", { visible: true });
const dropdowns = await page.$$(".SearchFilters_filterContainer .Dropdown");
await dropdowns[1].click();
await page.waitForSelector(".Dropdown_list--open button", { visible: true });
await page.evaluate(() => {
const dateButton = Array.from(document.querySelectorAll(".Dropdown_list--open button"))
.find(button => button.textContent.trim() === "Date");
if (dateButton) dateButton.click();
});
// Set range to 'Past Week'
await page.waitForSelector(".SearchFilters_filterContainer .Dropdown", { visible: true });
await dropdowns[2].click();
await page.waitForSelector(".Dropdown_list--open button", { visible: true });
await page.evaluate(() => {
const dateButton = Array.from(document.querySelectorAll(".Dropdown_list--open button"))
.find(button => button.textContent.trim() === "Past Week");
if (dateButton) dateButton.click();
});
// Perform search with the original query (AND logic by default)
await page.click("input[type=\"search\"]");
await page.type("input[type=\"search\"]", query);
await page.keyboard.press("Enter");
let allThreads = [];
let hasMorePages = true;
for (let i = 1; i <= pages && hasMorePages; i++) {
console.info(`Scraping page ${i}...`);
let pageThreads = await scrapePageThreads(page);
allThreads = [...allThreads, ...pageThreads];
console.log(`Threads found on page ${i}: ${pageThreads.length}`);
if (i < pages) {
hasMorePages = await goToNextPage(page, i);
if (!hasMorePages) {
console.log("No more pages available. Ending search.");
}
}
}
console.log(`Total threads scraped: ${allThreads.length}`);
// If there are no threads and no pagination, return an empty array
if (allThreads.length === 0) {
console.log("No results found. Returning empty array.");
return [];
}
return allThreads.map(thread => ({
source: thread.source,
url: thread.url ?? "",
title: thread.title ?? "",
date_published: thread.date_published ?? "",
}));
} catch (error) {
console.error("An error occurred:", error);
return [];
} finally {
if (browser) {
await browser.close();
}
}