feat: add news source adapters and AI article analysis
- Implement adapters for NewsAPI, GNews, NewsData, Serper Web Search, and Brave Search. - Add unified fetchNews aggregator with URL deduplication. - Wire OpenAI-compatible Kimi client for bias classification, ranking, summarization, and topic extraction. - Add AI analysis prompts, response parsing, and fallback logic. - Build ranker combining AI score, source credibility, and recency. - Create admin Discover page and API for manual news discovery.
This commit is contained in:
46
src/lib/news/index.ts
Normal file
46
src/lib/news/index.ts
Normal file
@@ -0,0 +1,46 @@
|
||||
import { FetchNewsOptions, RawArticle } from './types';
|
||||
import { fetchNewsApi } from './sources/newsapi';
|
||||
import { fetchGNews } from './sources/gnews';
|
||||
import { fetchWebSearch } from './sources/websearch';
|
||||
import { fetchNewsData } from './sources/newsdata';
|
||||
import { fetchBrave } from './sources/brave';
|
||||
|
||||
const sourceMap: Record<string, (opts: FetchNewsOptions) => Promise<RawArticle[]>> = {
|
||||
newsapi: fetchNewsApi,
|
||||
gnews: fetchGNews,
|
||||
websearch: fetchWebSearch,
|
||||
newsdata: fetchNewsData,
|
||||
brave: fetchBrave,
|
||||
};
|
||||
|
||||
export async function fetchNews(options: FetchNewsOptions): Promise<RawArticle[]> {
|
||||
const sources = options.sources && options.sources.length > 0
|
||||
? options.sources
|
||||
: Object.keys(sourceMap);
|
||||
|
||||
const results = await Promise.allSettled(
|
||||
sources.map(async (source) => {
|
||||
const fetcher = sourceMap[source.toLowerCase()];
|
||||
if (!fetcher) throw new Error(`Unknown news source: ${source}`);
|
||||
return fetcher(options);
|
||||
})
|
||||
);
|
||||
|
||||
const articles: RawArticle[] = [];
|
||||
results.forEach((result, index) => {
|
||||
if (result.status === 'fulfilled') {
|
||||
articles.push(...result.value);
|
||||
} else {
|
||||
console.error(`Source ${sources[index]} failed:`, result.reason);
|
||||
}
|
||||
});
|
||||
|
||||
// Deduplicate by URL
|
||||
const seen = new Set<string>();
|
||||
return articles.filter((article) => {
|
||||
const normalized = article.url.split('?')[0].toLowerCase();
|
||||
if (seen.has(normalized)) return false;
|
||||
seen.add(normalized);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
80
src/lib/news/sources/brave.ts
Normal file
80
src/lib/news/sources/brave.ts
Normal file
@@ -0,0 +1,80 @@
|
||||
import { FetchNewsOptions, RawArticle } from '../types';
|
||||
import { getActiveKey } from '@/lib/api-keys';
|
||||
|
||||
function parseBraveAge(age?: string): Date | undefined {
|
||||
if (!age) return undefined;
|
||||
// Brave returns relative strings like "1 hour ago", "2 days ago", "Jun 16, 2026"
|
||||
// We can only reliably parse the ISO-like or simple relative forms here.
|
||||
// For now, fall back to 'now' if parsing fails.
|
||||
try {
|
||||
const match = age.match(/(\d+)\s+(minute|hour|day|week|month|year)s?\s+ago/i);
|
||||
if (match) {
|
||||
const value = parseInt(match[1], 10);
|
||||
const unit = match[2].toLowerCase();
|
||||
const now = new Date();
|
||||
switch (unit) {
|
||||
case 'minute': now.setMinutes(now.getMinutes() - value); break;
|
||||
case 'hour': now.setHours(now.getHours() - value); break;
|
||||
case 'day': now.setDate(now.getDate() - value); break;
|
||||
case 'week': now.setDate(now.getDate() - value * 7); break;
|
||||
case 'month': now.setMonth(now.getMonth() - value); break;
|
||||
case 'year': now.setFullYear(now.getFullYear() - value); break;
|
||||
}
|
||||
return now;
|
||||
}
|
||||
const parsed = new Date(age);
|
||||
if (!isNaN(parsed.getTime())) return parsed;
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export async function fetchBrave(options: FetchNewsOptions): Promise<RawArticle[]> {
|
||||
const apiKey = await getActiveKey('BRAVE');
|
||||
if (!apiKey) {
|
||||
console.warn('Brave Search key not configured');
|
||||
return [];
|
||||
}
|
||||
|
||||
const params = new URLSearchParams({
|
||||
q: options.query,
|
||||
count: String(Math.min(options.maxResults || 10, 20)),
|
||||
search_lang: 'en',
|
||||
text_decorations: 'false',
|
||||
});
|
||||
|
||||
if (options.from) {
|
||||
params.set('freshness', `dtf:${options.from.toISOString().split('T')[0]}_9999-12-31`);
|
||||
}
|
||||
|
||||
const res = await fetch(`https://api.search.brave.com/res/v1/news/search?${params.toString()}`, {
|
||||
headers: {
|
||||
'X-Subscription-Token': apiKey,
|
||||
'Accept': 'application/json',
|
||||
},
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
const text = await res.text();
|
||||
throw new Error(`Brave search error ${res.status}: ${text}`);
|
||||
}
|
||||
|
||||
const data = await res.json();
|
||||
|
||||
return (data.results || []).map((item: Record<string, unknown>) => {
|
||||
const meta = item.meta as Record<string, unknown> | undefined;
|
||||
const metaUrl = meta?.url as Record<string, unknown> | undefined;
|
||||
return {
|
||||
title: String(item.title || ''),
|
||||
url: String(item.url || ''),
|
||||
sourceName: String(metaUrl?.hostname || item.source || 'Brave Search'),
|
||||
sourceApiCode: 'brave',
|
||||
publishedAt: parseBraveAge(item.age ? String(item.age) : undefined),
|
||||
description: item.description ? String(item.description) : undefined,
|
||||
content: undefined,
|
||||
imageUrl: metaUrl?.favicon ? String(metaUrl.favicon) : undefined,
|
||||
raw: item,
|
||||
};
|
||||
});
|
||||
}
|
||||
40
src/lib/news/sources/gnews.ts
Normal file
40
src/lib/news/sources/gnews.ts
Normal file
@@ -0,0 +1,40 @@
|
||||
import { FetchNewsOptions, RawArticle } from '../types';
|
||||
import { getActiveKey } from '@/lib/api-keys';
|
||||
|
||||
export async function fetchGNews(options: FetchNewsOptions): Promise<RawArticle[]> {
|
||||
const apiKey = await getActiveKey('GNEWS');
|
||||
if (!apiKey) {
|
||||
console.warn('GNews key not configured');
|
||||
return [];
|
||||
}
|
||||
|
||||
const url = new URL('https://gnews.io/api/v4/search');
|
||||
url.searchParams.set('q', options.query);
|
||||
url.searchParams.set('apikey', apiKey);
|
||||
url.searchParams.set('max', String(Math.min(options.maxResults || 20, 100)));
|
||||
url.searchParams.set('lang', 'en');
|
||||
url.searchParams.set('sortby', 'publishedAt');
|
||||
|
||||
if (options.from) url.searchParams.set('from', options.from.toISOString());
|
||||
if (options.to) url.searchParams.set('to', options.to.toISOString());
|
||||
|
||||
const res = await fetch(url.toString());
|
||||
if (!res.ok) {
|
||||
const text = await res.text();
|
||||
throw new Error(`GNews error ${res.status}: ${text}`);
|
||||
}
|
||||
|
||||
const data = await res.json();
|
||||
|
||||
return (data.articles || []).map((article: Record<string, unknown>) => ({
|
||||
title: String(article.title || ''),
|
||||
url: String(article.url || ''),
|
||||
sourceName: String((article.source as Record<string, unknown>)?.name || 'GNews'),
|
||||
sourceApiCode: 'gnews',
|
||||
publishedAt: article.publishedAt ? new Date(String(article.publishedAt)) : undefined,
|
||||
description: article.description ? String(article.description) : undefined,
|
||||
content: article.content ? String(article.content) : undefined,
|
||||
imageUrl: article.image ? String(article.image) : undefined,
|
||||
raw: article,
|
||||
}));
|
||||
}
|
||||
43
src/lib/news/sources/newsapi.ts
Normal file
43
src/lib/news/sources/newsapi.ts
Normal file
@@ -0,0 +1,43 @@
|
||||
import { FetchNewsOptions, RawArticle } from '../types';
|
||||
import { getActiveKey } from '@/lib/api-keys';
|
||||
|
||||
export async function fetchNewsApi(options: FetchNewsOptions): Promise<RawArticle[]> {
|
||||
const apiKey = await getActiveKey('NEWSAPI');
|
||||
if (!apiKey) {
|
||||
console.warn('NewsAPI key not configured');
|
||||
return [];
|
||||
}
|
||||
|
||||
const url = new URL('https://newsapi.org/v2/everything');
|
||||
url.searchParams.set('q', options.query);
|
||||
url.searchParams.set('apiKey', apiKey);
|
||||
url.searchParams.set('pageSize', String(Math.min(options.maxResults || 20, 100)));
|
||||
url.searchParams.set('sortBy', 'publishedAt');
|
||||
url.searchParams.set('language', 'en');
|
||||
|
||||
if (options.from) url.searchParams.set('from', options.from.toISOString());
|
||||
if (options.to) url.searchParams.set('to', options.to.toISOString());
|
||||
|
||||
const res = await fetch(url.toString());
|
||||
if (!res.ok) {
|
||||
const text = await res.text();
|
||||
throw new Error(`NewsAPI error ${res.status}: ${text}`);
|
||||
}
|
||||
|
||||
const data = await res.json();
|
||||
if (data.status !== 'ok') {
|
||||
throw new Error(`NewsAPI error: ${data.message || 'unknown'}`);
|
||||
}
|
||||
|
||||
return (data.articles || []).map((article: Record<string, unknown>) => ({
|
||||
title: String(article.title || ''),
|
||||
url: String(article.url || ''),
|
||||
sourceName: String((article.source as Record<string, unknown>)?.name || 'NewsAPI'),
|
||||
sourceApiCode: 'newsapi',
|
||||
publishedAt: article.publishedAt ? new Date(String(article.publishedAt)) : undefined,
|
||||
description: article.description ? String(article.description) : undefined,
|
||||
content: article.content ? String(article.content) : undefined,
|
||||
imageUrl: article.urlToImage ? String(article.urlToImage) : undefined,
|
||||
raw: article,
|
||||
}));
|
||||
}
|
||||
42
src/lib/news/sources/newsdata.ts
Normal file
42
src/lib/news/sources/newsdata.ts
Normal file
@@ -0,0 +1,42 @@
|
||||
import { FetchNewsOptions, RawArticle } from '../types';
|
||||
import { getActiveKey } from '@/lib/api-keys';
|
||||
|
||||
export async function fetchNewsData(options: FetchNewsOptions): Promise<RawArticle[]> {
|
||||
const apiKey = await getActiveKey('NEWSDATA');
|
||||
if (!apiKey) {
|
||||
console.warn('NewsData key not configured');
|
||||
return [];
|
||||
}
|
||||
|
||||
const url = new URL('https://newsdata.io/api/1/latest');
|
||||
url.searchParams.set('q', options.query);
|
||||
url.searchParams.set('apikey', apiKey);
|
||||
url.searchParams.set('size', String(Math.min(options.maxResults || 10, 50)));
|
||||
url.searchParams.set('language', 'en');
|
||||
|
||||
if (options.from) url.searchParams.set('from_date', options.from.toISOString().split('T')[0]);
|
||||
if (options.to) url.searchParams.set('to_date', options.to.toISOString().split('T')[0]);
|
||||
|
||||
const res = await fetch(url.toString());
|
||||
if (!res.ok) {
|
||||
const text = await res.text();
|
||||
throw new Error(`NewsData error ${res.status}: ${text}`);
|
||||
}
|
||||
|
||||
const data = await res.json();
|
||||
if (data.status !== 'success') {
|
||||
throw new Error(`NewsData error: ${data.message || data.status || 'unknown'}`);
|
||||
}
|
||||
|
||||
return (data.results || []).map((article: Record<string, unknown>) => ({
|
||||
title: String(article.title || ''),
|
||||
url: String(article.link || ''),
|
||||
sourceName: String(article.source_id || 'NewsData'),
|
||||
sourceApiCode: 'newsdata',
|
||||
publishedAt: article.pubDate ? new Date(String(article.pubDate)) : undefined,
|
||||
description: article.description ? String(article.description) : undefined,
|
||||
content: article.content ? String(article.content) : undefined,
|
||||
imageUrl: article.image_url ? String(article.image_url) : undefined,
|
||||
raw: article,
|
||||
}));
|
||||
}
|
||||
42
src/lib/news/sources/websearch.ts
Normal file
42
src/lib/news/sources/websearch.ts
Normal file
@@ -0,0 +1,42 @@
|
||||
import { FetchNewsOptions, RawArticle } from '../types';
|
||||
import { getActiveKey } from '@/lib/api-keys';
|
||||
|
||||
export async function fetchWebSearch(options: FetchNewsOptions): Promise<RawArticle[]> {
|
||||
const apiKey = await getActiveKey('WEBSEARCH');
|
||||
if (!apiKey) {
|
||||
console.warn('Web search key not configured');
|
||||
return [];
|
||||
}
|
||||
|
||||
const res = await fetch('https://google.serper.dev/search', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'X-API-KEY': apiKey,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
q: options.query,
|
||||
num: Math.min(options.maxResults || 10, 20),
|
||||
tbs: options.from ? `cdr:1,cd_min:${options.from.toISOString().split('T')[0]}` : undefined,
|
||||
}),
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
const text = await res.text();
|
||||
throw new Error(`Web search error ${res.status}: ${text}`);
|
||||
}
|
||||
|
||||
const data = await res.json();
|
||||
|
||||
return (data.news || data.organic || []).map((item: Record<string, unknown>) => ({
|
||||
title: String(item.title || ''),
|
||||
url: String(item.link || item.url || ''),
|
||||
sourceName: String(item.source || 'Web Search'),
|
||||
sourceApiCode: 'websearch',
|
||||
publishedAt: item.date ? new Date(String(item.date)) : undefined,
|
||||
description: item.snippet ? String(item.snippet) : undefined,
|
||||
content: undefined,
|
||||
imageUrl: item.imageUrl ? String(item.imageUrl) : undefined,
|
||||
raw: item,
|
||||
}));
|
||||
}
|
||||
19
src/lib/news/types.ts
Normal file
19
src/lib/news/types.ts
Normal file
@@ -0,0 +1,19 @@
|
||||
export type RawArticle = {
|
||||
title: string;
|
||||
url: string;
|
||||
sourceName: string;
|
||||
sourceApiCode: string;
|
||||
publishedAt?: Date;
|
||||
description?: string;
|
||||
content?: string;
|
||||
imageUrl?: string;
|
||||
raw: Record<string, unknown>;
|
||||
};
|
||||
|
||||
export type FetchNewsOptions = {
|
||||
query: string;
|
||||
sources?: string[];
|
||||
maxResults?: number;
|
||||
from?: Date;
|
||||
to?: Date;
|
||||
};
|
||||
Reference in New Issue
Block a user