feat: add news source adapters and AI article analysis

- Implement adapters for NewsAPI, GNews, NewsData, Serper Web Search,
  and Brave Search.
- Add unified fetchNews aggregator with URL deduplication.
- Wire OpenAI-compatible Kimi client for bias classification, ranking,
  summarization, and topic extraction.
- Add AI analysis prompts, response parsing, and fallback logic.
- Build ranker combining AI score, source credibility, and recency.
- Create admin Discover page and API for manual news discovery.
This commit is contained in:
hermes
2026-06-17 04:32:25 +00:00
parent e086141620
commit 3e57802c6e
14 changed files with 897 additions and 0 deletions

46
src/lib/news/index.ts Normal file
View File

@@ -0,0 +1,46 @@
import { FetchNewsOptions, RawArticle } from './types';
import { fetchNewsApi } from './sources/newsapi';
import { fetchGNews } from './sources/gnews';
import { fetchWebSearch } from './sources/websearch';
import { fetchNewsData } from './sources/newsdata';
import { fetchBrave } from './sources/brave';
const sourceMap: Record<string, (opts: FetchNewsOptions) => Promise<RawArticle[]>> = {
newsapi: fetchNewsApi,
gnews: fetchGNews,
websearch: fetchWebSearch,
newsdata: fetchNewsData,
brave: fetchBrave,
};
export async function fetchNews(options: FetchNewsOptions): Promise<RawArticle[]> {
const sources = options.sources && options.sources.length > 0
? options.sources
: Object.keys(sourceMap);
const results = await Promise.allSettled(
sources.map(async (source) => {
const fetcher = sourceMap[source.toLowerCase()];
if (!fetcher) throw new Error(`Unknown news source: ${source}`);
return fetcher(options);
})
);
const articles: RawArticle[] = [];
results.forEach((result, index) => {
if (result.status === 'fulfilled') {
articles.push(...result.value);
} else {
console.error(`Source ${sources[index]} failed:`, result.reason);
}
});
// Deduplicate by URL
const seen = new Set<string>();
return articles.filter((article) => {
const normalized = article.url.split('?')[0].toLowerCase();
if (seen.has(normalized)) return false;
seen.add(normalized);
return true;
});
}

View File

@@ -0,0 +1,80 @@
import { FetchNewsOptions, RawArticle } from '../types';
import { getActiveKey } from '@/lib/api-keys';
function parseBraveAge(age?: string): Date | undefined {
if (!age) return undefined;
// Brave returns relative strings like "1 hour ago", "2 days ago", "Jun 16, 2026"
// We can only reliably parse the ISO-like or simple relative forms here.
// For now, fall back to 'now' if parsing fails.
try {
const match = age.match(/(\d+)\s+(minute|hour|day|week|month|year)s?\s+ago/i);
if (match) {
const value = parseInt(match[1], 10);
const unit = match[2].toLowerCase();
const now = new Date();
switch (unit) {
case 'minute': now.setMinutes(now.getMinutes() - value); break;
case 'hour': now.setHours(now.getHours() - value); break;
case 'day': now.setDate(now.getDate() - value); break;
case 'week': now.setDate(now.getDate() - value * 7); break;
case 'month': now.setMonth(now.getMonth() - value); break;
case 'year': now.setFullYear(now.getFullYear() - value); break;
}
return now;
}
const parsed = new Date(age);
if (!isNaN(parsed.getTime())) return parsed;
} catch {
// ignore
}
return undefined;
}
export async function fetchBrave(options: FetchNewsOptions): Promise<RawArticle[]> {
const apiKey = await getActiveKey('BRAVE');
if (!apiKey) {
console.warn('Brave Search key not configured');
return [];
}
const params = new URLSearchParams({
q: options.query,
count: String(Math.min(options.maxResults || 10, 20)),
search_lang: 'en',
text_decorations: 'false',
});
if (options.from) {
params.set('freshness', `dtf:${options.from.toISOString().split('T')[0]}_9999-12-31`);
}
const res = await fetch(`https://api.search.brave.com/res/v1/news/search?${params.toString()}`, {
headers: {
'X-Subscription-Token': apiKey,
'Accept': 'application/json',
},
});
if (!res.ok) {
const text = await res.text();
throw new Error(`Brave search error ${res.status}: ${text}`);
}
const data = await res.json();
return (data.results || []).map((item: Record<string, unknown>) => {
const meta = item.meta as Record<string, unknown> | undefined;
const metaUrl = meta?.url as Record<string, unknown> | undefined;
return {
title: String(item.title || ''),
url: String(item.url || ''),
sourceName: String(metaUrl?.hostname || item.source || 'Brave Search'),
sourceApiCode: 'brave',
publishedAt: parseBraveAge(item.age ? String(item.age) : undefined),
description: item.description ? String(item.description) : undefined,
content: undefined,
imageUrl: metaUrl?.favicon ? String(metaUrl.favicon) : undefined,
raw: item,
};
});
}

View File

@@ -0,0 +1,40 @@
import { FetchNewsOptions, RawArticle } from '../types';
import { getActiveKey } from '@/lib/api-keys';
export async function fetchGNews(options: FetchNewsOptions): Promise<RawArticle[]> {
const apiKey = await getActiveKey('GNEWS');
if (!apiKey) {
console.warn('GNews key not configured');
return [];
}
const url = new URL('https://gnews.io/api/v4/search');
url.searchParams.set('q', options.query);
url.searchParams.set('apikey', apiKey);
url.searchParams.set('max', String(Math.min(options.maxResults || 20, 100)));
url.searchParams.set('lang', 'en');
url.searchParams.set('sortby', 'publishedAt');
if (options.from) url.searchParams.set('from', options.from.toISOString());
if (options.to) url.searchParams.set('to', options.to.toISOString());
const res = await fetch(url.toString());
if (!res.ok) {
const text = await res.text();
throw new Error(`GNews error ${res.status}: ${text}`);
}
const data = await res.json();
return (data.articles || []).map((article: Record<string, unknown>) => ({
title: String(article.title || ''),
url: String(article.url || ''),
sourceName: String((article.source as Record<string, unknown>)?.name || 'GNews'),
sourceApiCode: 'gnews',
publishedAt: article.publishedAt ? new Date(String(article.publishedAt)) : undefined,
description: article.description ? String(article.description) : undefined,
content: article.content ? String(article.content) : undefined,
imageUrl: article.image ? String(article.image) : undefined,
raw: article,
}));
}

View File

@@ -0,0 +1,43 @@
import { FetchNewsOptions, RawArticle } from '../types';
import { getActiveKey } from '@/lib/api-keys';
export async function fetchNewsApi(options: FetchNewsOptions): Promise<RawArticle[]> {
const apiKey = await getActiveKey('NEWSAPI');
if (!apiKey) {
console.warn('NewsAPI key not configured');
return [];
}
const url = new URL('https://newsapi.org/v2/everything');
url.searchParams.set('q', options.query);
url.searchParams.set('apiKey', apiKey);
url.searchParams.set('pageSize', String(Math.min(options.maxResults || 20, 100)));
url.searchParams.set('sortBy', 'publishedAt');
url.searchParams.set('language', 'en');
if (options.from) url.searchParams.set('from', options.from.toISOString());
if (options.to) url.searchParams.set('to', options.to.toISOString());
const res = await fetch(url.toString());
if (!res.ok) {
const text = await res.text();
throw new Error(`NewsAPI error ${res.status}: ${text}`);
}
const data = await res.json();
if (data.status !== 'ok') {
throw new Error(`NewsAPI error: ${data.message || 'unknown'}`);
}
return (data.articles || []).map((article: Record<string, unknown>) => ({
title: String(article.title || ''),
url: String(article.url || ''),
sourceName: String((article.source as Record<string, unknown>)?.name || 'NewsAPI'),
sourceApiCode: 'newsapi',
publishedAt: article.publishedAt ? new Date(String(article.publishedAt)) : undefined,
description: article.description ? String(article.description) : undefined,
content: article.content ? String(article.content) : undefined,
imageUrl: article.urlToImage ? String(article.urlToImage) : undefined,
raw: article,
}));
}

View File

@@ -0,0 +1,42 @@
import { FetchNewsOptions, RawArticle } from '../types';
import { getActiveKey } from '@/lib/api-keys';
export async function fetchNewsData(options: FetchNewsOptions): Promise<RawArticle[]> {
const apiKey = await getActiveKey('NEWSDATA');
if (!apiKey) {
console.warn('NewsData key not configured');
return [];
}
const url = new URL('https://newsdata.io/api/1/latest');
url.searchParams.set('q', options.query);
url.searchParams.set('apikey', apiKey);
url.searchParams.set('size', String(Math.min(options.maxResults || 10, 50)));
url.searchParams.set('language', 'en');
if (options.from) url.searchParams.set('from_date', options.from.toISOString().split('T')[0]);
if (options.to) url.searchParams.set('to_date', options.to.toISOString().split('T')[0]);
const res = await fetch(url.toString());
if (!res.ok) {
const text = await res.text();
throw new Error(`NewsData error ${res.status}: ${text}`);
}
const data = await res.json();
if (data.status !== 'success') {
throw new Error(`NewsData error: ${data.message || data.status || 'unknown'}`);
}
return (data.results || []).map((article: Record<string, unknown>) => ({
title: String(article.title || ''),
url: String(article.link || ''),
sourceName: String(article.source_id || 'NewsData'),
sourceApiCode: 'newsdata',
publishedAt: article.pubDate ? new Date(String(article.pubDate)) : undefined,
description: article.description ? String(article.description) : undefined,
content: article.content ? String(article.content) : undefined,
imageUrl: article.image_url ? String(article.image_url) : undefined,
raw: article,
}));
}

View File

@@ -0,0 +1,42 @@
import { FetchNewsOptions, RawArticle } from '../types';
import { getActiveKey } from '@/lib/api-keys';
export async function fetchWebSearch(options: FetchNewsOptions): Promise<RawArticle[]> {
const apiKey = await getActiveKey('WEBSEARCH');
if (!apiKey) {
console.warn('Web search key not configured');
return [];
}
const res = await fetch('https://google.serper.dev/search', {
method: 'POST',
headers: {
'X-API-KEY': apiKey,
'Content-Type': 'application/json',
},
body: JSON.stringify({
q: options.query,
num: Math.min(options.maxResults || 10, 20),
tbs: options.from ? `cdr:1,cd_min:${options.from.toISOString().split('T')[0]}` : undefined,
}),
});
if (!res.ok) {
const text = await res.text();
throw new Error(`Web search error ${res.status}: ${text}`);
}
const data = await res.json();
return (data.news || data.organic || []).map((item: Record<string, unknown>) => ({
title: String(item.title || ''),
url: String(item.link || item.url || ''),
sourceName: String(item.source || 'Web Search'),
sourceApiCode: 'websearch',
publishedAt: item.date ? new Date(String(item.date)) : undefined,
description: item.snippet ? String(item.snippet) : undefined,
content: undefined,
imageUrl: item.imageUrl ? String(item.imageUrl) : undefined,
raw: item,
}));
}

19
src/lib/news/types.ts Normal file
View File

@@ -0,0 +1,19 @@
export type RawArticle = {
title: string;
url: string;
sourceName: string;
sourceApiCode: string;
publishedAt?: Date;
description?: string;
content?: string;
imageUrl?: string;
raw: Record<string, unknown>;
};
export type FetchNewsOptions = {
query: string;
sources?: string[];
maxResults?: number;
from?: Date;
to?: Date;
};