From 3e57802c6e240ddf8ff88d8a92542b6810f52cdf Mon Sep 17 00:00:00 2001 From: hermes Date: Wed, 17 Jun 2026 04:32:25 +0000 Subject: [PATCH] feat: add news source adapters and AI article analysis - Implement adapters for NewsAPI, GNews, NewsData, Serper Web Search, and Brave Search. - Add unified fetchNews aggregator with URL deduplication. - Wire OpenAI-compatible Kimi client for bias classification, ranking, summarization, and topic extraction. - Add AI analysis prompts, response parsing, and fallback logic. - Build ranker combining AI score, source credibility, and recency. - Create admin Discover page and API for manual news discovery. --- src/app/admin/discover/page.tsx | 233 ++++++++++++++++++++++++++++ src/app/api/admin/discover/route.ts | 58 +++++++ src/lib/ai/client.ts | 17 ++ src/lib/ai/processor.ts | 95 ++++++++++++ src/lib/ai/prompts.ts | 55 +++++++ src/lib/ai/ranker.ts | 21 +++ src/lib/articles.ts | 106 +++++++++++++ src/lib/news/index.ts | 46 ++++++ src/lib/news/sources/brave.ts | 80 ++++++++++ src/lib/news/sources/gnews.ts | 40 +++++ src/lib/news/sources/newsapi.ts | 43 +++++ src/lib/news/sources/newsdata.ts | 42 +++++ src/lib/news/sources/websearch.ts | 42 +++++ src/lib/news/types.ts | 19 +++ 14 files changed, 897 insertions(+) create mode 100644 src/app/admin/discover/page.tsx create mode 100644 src/app/api/admin/discover/route.ts create mode 100644 src/lib/ai/client.ts create mode 100644 src/lib/ai/processor.ts create mode 100644 src/lib/ai/prompts.ts create mode 100644 src/lib/ai/ranker.ts create mode 100644 src/lib/articles.ts create mode 100644 src/lib/news/index.ts create mode 100644 src/lib/news/sources/brave.ts create mode 100644 src/lib/news/sources/gnews.ts create mode 100644 src/lib/news/sources/newsapi.ts create mode 100644 src/lib/news/sources/newsdata.ts create mode 100644 src/lib/news/sources/websearch.ts create mode 100644 src/lib/news/types.ts diff --git a/src/app/admin/discover/page.tsx b/src/app/admin/discover/page.tsx new file mode 100644 index 0000000..4cd44a8 --- /dev/null +++ b/src/app/admin/discover/page.tsx @@ -0,0 +1,233 @@ +'use client'; + +import { useEffect, useState } from 'react'; +import { Bias } from '@prisma/client'; + +const availableSources = [ + { code: 'newsapi', name: 'NewsAPI' }, + { code: 'gnews', name: 'GNews' }, + { code: 'websearch', name: 'Web Search (Serper)' }, +]; + +const biasColors: Record = { + LEFT: 'bg-blue-100 text-blue-800 dark:bg-blue-950 dark:text-blue-300', + CENTER: 'bg-purple-100 text-purple-800 dark:bg-purple-950 dark:text-purple-300', + RIGHT: 'bg-red-100 text-red-800 dark:bg-red-950 dark:text-red-300', +}; + +type DiscoveredArticle = { + success: boolean; + article?: { + id: string; + title: string; + summary: string; + url: string; + sourceName: string; + primaryBias: Bias; + politicalTags: string[]; + rankScore: number; + credibilityScore: number; + topics: string[]; + status: string; + }; + error?: string; +}; + +export default function DiscoverPage() { + const [query, setQuery] = useState(''); + const [selectedSources, setSelectedSources] = useState(['newsapi', 'gnews']); + const [maxResults, setMaxResults] = useState(10); + const [autoPublish, setAutoPublish] = useState(false); + const [loading, setLoading] = useState(false); + const [results, setResults] = useState([]); + const [stats, setStats] = useState<{ found: number; processed: number; failed: number } | null>(null); + + const toggleSource = (code: string) => { + setSelectedSources((prev) => + prev.includes(code) ? prev.filter((c) => c !== code) : [...prev, code] + ); + }; + + const handleDiscover = async (e: React.FormEvent) => { + e.preventDefault(); + if (selectedSources.length === 0) { + alert('Select at least one source'); + return; + } + setLoading(true); + setResults([]); + setStats(null); + + try { + const res = await fetch('/api/admin/discover', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + query, + sources: selectedSources, + maxResults, + autoPublish, + }), + }); + + const data = await res.json(); + if (!res.ok) { + alert(data.error || 'Discovery failed'); + return; + } + + setResults(data.results || []); + setStats({ found: data.found, processed: data.processed, failed: data.failed }); + } finally { + setLoading(false); + } + }; + + return ( +
+

Discover News

+ +
+
+
+ + setQuery(e.target.value)} + className="w-full px-3 py-2 border rounded-lg dark:bg-neutral-800 dark:border-neutral-700" + placeholder="e.g. US politics, climate policy, Supreme Court" + required + /> +
+ +
+ +
+ {availableSources.map((s) => ( + + ))} +
+
+ +
+
+ + setMaxResults(Number(e.target.value))} + className="w-full px-3 py-2 border rounded-lg dark:bg-neutral-800 dark:border-neutral-700" + /> +
+
+ setAutoPublish(e.target.checked)} + /> + +
+
+ +
+
+
+
+ + {stats && ( +
+ Found {stats.found} raw articles. Processed {stats.processed}, failed {stats.failed}. +
+ )} + +
+ {results.map((result, idx) => { + if (!result.success || !result.article) { + return ( +
+ Failed: {result.error} +
+ ); + } + + const a = result.article; + return ( +
+
+

+ + {a.title} + +

+ + {a.primaryBias} + +
+

+ {a.summary} +

+
+ {a.politicalTags.map((tag) => ( + + {tag} + + ))} + {a.topics.slice(0, 5).map((topic) => ( + + {topic} + + ))} +
+
+ Source: {a.sourceName} + Rank: {a.rankScore}/10 + Credibility: {a.credibilityScore}/10 + Status: {a.status} +
+
+ ); + })} +
+
+ ); +} diff --git a/src/app/api/admin/discover/route.ts b/src/app/api/admin/discover/route.ts new file mode 100644 index 0000000..8e5e5e0 --- /dev/null +++ b/src/app/api/admin/discover/route.ts @@ -0,0 +1,58 @@ +import { NextResponse } from 'next/server'; +import { fetchNews } from '@/lib/news'; +import { processArticle } from '@/lib/ai/processor'; +import { saveProcessedArticle } from '@/lib/articles'; + +export async function POST(request: Request) { + try { + const body = await request.json(); + const { query, sources, maxResults, autoPublish } = body; + + if (!query || typeof query !== 'string') { + return NextResponse.json({ error: 'Query is required' }, { status: 400 }); + } + + const rawArticles = await fetchNews({ + query, + sources: sources || undefined, + maxResults: maxResults || 20, + }); + + const processed = await Promise.allSettled( + rawArticles.map(async (raw) => { + const analyzed = await processArticle(raw); + const status = autoPublish ? 'PUBLISHED' : 'DRAFT'; + const saved = await saveProcessedArticle(analyzed, status); + return { + ...analyzed, + id: saved.id, + status, + politicalTags: analyzed.politicalTags, + topics: analyzed.topics, + }; + }) + ); + + const results = processed.map((result) => + result.status === 'fulfilled' + ? { success: true, article: result.value } + : { success: false, error: String(result.reason) } + ); + + const succeeded = results.filter((r) => r.success); + const failed = results.filter((r) => !r.success); + + return NextResponse.json({ + found: rawArticles.length, + processed: succeeded.length, + failed: failed.length, + results, + }); + } catch (error) { + console.error('Discover error:', error); + return NextResponse.json( + { error: String(error) }, + { status: 500 } + ); + } +} diff --git a/src/lib/ai/client.ts b/src/lib/ai/client.ts new file mode 100644 index 0000000..6039654 --- /dev/null +++ b/src/lib/ai/client.ts @@ -0,0 +1,17 @@ +import OpenAI from 'openai'; +import { getActiveKey } from '../api-keys'; + +export async function getAiClient(): Promise { + const apiKey = await getActiveKey('KIMI'); + const baseURL = process.env.KIMI_API_URL || 'https://api.moonshot.ai/v1'; + + if (!apiKey) { + throw new Error('Kimi API key not configured. Add one in Admin > API Keys.'); + } + + return new OpenAI({ apiKey, baseURL, timeout: 60_000, maxRetries: 1 }); +} + +export function getAiModel(): string { + return process.env.KIMI_MODEL || 'kimi-k2.5'; +} diff --git a/src/lib/ai/processor.ts b/src/lib/ai/processor.ts new file mode 100644 index 0000000..5386b8c --- /dev/null +++ b/src/lib/ai/processor.ts @@ -0,0 +1,95 @@ +import { RawArticle } from '../news/types'; +import { getAiClient, getAiModel } from './client'; +import { buildArticleAnalysisPrompt } from './prompts'; +import { Bias } from '@prisma/client'; + +export type ProcessedArticle = { + title: string; + summary: string; + url: string; + sourceName: string; + sourceApiCode: string; + publishedAt?: Date; + imageUrl?: string; + primaryBias: Bias; + biasConfidence: number; + politicalTags: string[]; + rankScore: number; + credibilityScore: number; + topics: string[]; + aiExplanation: string; + raw: Record; +}; + +function safeParseBias(value: string): Bias { + const upper = String(value).toUpperCase(); + if (upper === 'LEFT' || upper === 'CENTER' || upper === 'RIGHT') { + return upper as Bias; + } + return 'CENTER'; +} + +function safeParseNumber(value: unknown, fallback: number, min = 0, max = 10): number { + const num = Number(value); + if (Number.isNaN(num)) return fallback; + return Math.max(min, Math.min(max, num)); +} + +export async function processArticle( + raw: RawArticle, + options?: { model?: string } +): Promise { + const client = await getAiClient(); + const prompt = buildArticleAnalysisPrompt({ + title: raw.title, + sourceName: raw.sourceName, + description: raw.description, + content: raw.content, + url: raw.url, + }); + + const res = await client.chat.completions.create({ + model: options?.model || getAiModel(), + messages: [{ role: 'user', content: prompt }], + }); + + const content = res.choices[0]?.message?.content?.trim() || '{}'; + let parsed: Record = {}; + + try { + const cleaned = content.replace(/^```json\s*|\s*```$/g, ''); + parsed = JSON.parse(cleaned); + } catch (err) { + console.error('Failed to parse AI response:', content, err); + parsed = {}; + } + + const politicalTags = Array.isArray(parsed.politicalTags) + ? parsed.politicalTags.map(String).filter((t) => ['liberal', 'moderate', 'conservative'].includes(t.toLowerCase())) + : []; + + if (politicalTags.length === 0) { + const bias = safeParseBias(String(parsed.primaryBias)); + if (bias === 'LEFT') politicalTags.push('liberal'); + if (bias === 'CENTER') politicalTags.push('moderate'); + if (bias === 'RIGHT') politicalTags.push('conservative'); + } + + return { + title: raw.title, + summary: String(parsed.summary || raw.description || ''), + url: raw.url, + sourceName: raw.sourceName, + sourceApiCode: raw.sourceApiCode, + publishedAt: raw.publishedAt, + imageUrl: raw.imageUrl, + primaryBias: safeParseBias(parsed.primaryBias as string), + biasConfidence: safeParseNumber(parsed.biasConfidence, 0.5, 0, 1), + politicalTags, + rankScore: safeParseNumber(parsed.rankScore, 5, 1, 10), + credibilityScore: safeParseNumber(parsed.credibilityScore, 5, 1, 10), + topics: Array.isArray(parsed.topics) ? parsed.topics.map(String) : [], + aiExplanation: String(parsed.explanation || ''), + raw: raw.raw, + }; +} diff --git a/src/lib/ai/prompts.ts b/src/lib/ai/prompts.ts new file mode 100644 index 0000000..3642eeb --- /dev/null +++ b/src/lib/ai/prompts.ts @@ -0,0 +1,55 @@ +export function buildArticleAnalysisPrompt(article: { + title: string; + sourceName: string; + description?: string; + content?: string; + url: string; +}): string { + const text = [article.title, article.description, article.content] + .filter(Boolean) + .join('\n\n'); + + return `Analyze the following news article and return ONLY a JSON object with no markdown formatting. + +Article URL: ${article.url} +Source: ${article.sourceName} +Text: +""" +${text.slice(0, 12000)} +""" + +Return JSON in this exact shape: +{ + "summary": "A concise 2-3 sentence summary of the article.", + "primaryBias": "LEFT" | "CENTER" | "RIGHT", + "biasConfidence": 0.0 to 1.0, + "politicalTags": ["liberal" | "moderate" | "conservative"], + "rankScore": 1 to 10, + "credibilityScore": 1 to 10, + "topics": ["topic1", "topic2"], + "explanation": "One sentence explaining the bias classification." +} + +Definitions: +- LEFT: favors progressive/liberal policy, Democrats, or left-wing framing. +- CENTER: neutral, balanced, wire-service, or mainstream non-partisan framing. +- RIGHT: favors conservative/traditional policy, Republicans, or right-wing framing. +politicalTags should include one or more of: liberal, moderate, conservative. +rankScore reflects how important/significant the story is (1 = trivial, 10 = major). +credibilityScore reflects source reliability (1 = unreliable, 10 = highly credible).`; +} + +export function buildNewsSearchPrompt(query: string): string { + return `Find recent news articles about "${query}". Return ONLY a JSON array of objects with no markdown formatting. + +Each object must have: +{ + "title": "Article title", + "url": "Article URL", + "sourceName": "Source name", + "publishedAt": "ISO date string or null", + "description": "Brief description or snippet" +} + +Return at most 10 articles. If you cannot browse the live web, return an empty array [].`; +} diff --git a/src/lib/ai/ranker.ts b/src/lib/ai/ranker.ts new file mode 100644 index 0000000..950eedc --- /dev/null +++ b/src/lib/ai/ranker.ts @@ -0,0 +1,21 @@ +export function computeFinalRankScore({ + aiRankScore, + credibilityScore, + publishedAt, +}: { + aiRankScore: number; + credibilityScore: number; + publishedAt?: Date; +}): number { + // Base score: 60% AI rank, 40% credibility + let score = aiRankScore * 0.6 + credibilityScore * 0.4; + + // Recency boost: up to +1.5 for articles published in last 24h, decaying over 7 days + if (publishedAt) { + const ageHours = (Date.now() - publishedAt.getTime()) / (1000 * 60 * 60); + const recencyBoost = Math.max(0, 1.5 * (1 - ageHours / (24 * 7))); + score += recencyBoost; + } + + return Math.min(10, Math.max(1, Number(score.toFixed(2)))); +} diff --git a/src/lib/articles.ts b/src/lib/articles.ts new file mode 100644 index 0000000..735cd05 --- /dev/null +++ b/src/lib/articles.ts @@ -0,0 +1,106 @@ +import { prisma } from './db'; +import { ProcessedArticle } from './ai/processor'; +import { computeFinalRankScore } from './ai/ranker'; +import { ArticleStatus, Bias, Prisma } from '@prisma/client'; + +export async function findOrCreateSource(apiCode: string, name: string) { + return prisma.source.upsert({ + where: { apiCode }, + update: {}, + create: { apiCode, name }, + }); +} + +export async function saveProcessedArticle( + processed: ProcessedArticle, + status: ArticleStatus = 'DRAFT' +) { + const source = await findOrCreateSource(processed.sourceApiCode, processed.sourceName); + + const existing = await prisma.article.findUnique({ + where: { url: processed.url }, + }); + + const finalRank = computeFinalRankScore({ + aiRankScore: processed.rankScore, + credibilityScore: processed.credibilityScore, + publishedAt: processed.publishedAt, + }); + + const data: Prisma.ArticleUncheckedCreateInput = { + title: processed.title, + summary: processed.summary, + url: processed.url, + sourceId: source.id, + imageUrl: processed.imageUrl, + publishedAt: processed.publishedAt, + primaryBias: processed.primaryBias as Bias, + biasConfidence: processed.biasConfidence, + politicalTags: JSON.stringify(processed.politicalTags), + rankScore: finalRank, + credibilityScore: processed.credibilityScore, + status, + topics: JSON.stringify(processed.topics), + rawData: JSON.stringify(processed.raw), + aiExplanation: processed.aiExplanation, + }; + + if (existing) { + return prisma.article.update({ + where: { id: existing.id }, + data: { + ...data, + id: undefined, + url: undefined, + }, + }); + } + + return prisma.article.create({ data }); +} + +export async function getPublishedArticles(filters?: { + bias?: Bias; + query?: string; + sourceId?: string; + topic?: string; + from?: Date; + to?: Date; + limit?: number; + offset?: number; +}) { + const where: Prisma.ArticleWhereInput = { status: 'PUBLISHED' }; + + if (filters?.bias) where.primaryBias = filters.bias; + if (filters?.sourceId) where.sourceId = filters.sourceId; + if (filters?.topic) { + where.topics = { contains: `"${filters.topic}"` }; + } + if (filters?.from || filters?.to) { + where.publishedAt = {}; + if (filters.from) where.publishedAt.gte = filters.from; + if (filters.to) where.publishedAt.lte = filters.to; + } + if (filters?.query) { + where.OR = [ + { title: { contains: filters.query } }, + { summary: { contains: filters.query } }, + { topics: { contains: filters.query } }, + ]; + } + + return prisma.article.findMany({ + where, + orderBy: [{ rankScore: 'desc' }, { publishedAt: 'desc' }], + take: filters?.limit ?? 50, + skip: filters?.offset ?? 0, + include: { source: true }, + }); +} + +export async function getArticleById(id: string) { + return prisma.article.findUnique({ + where: { id }, + include: { source: true }, + }); +} diff --git a/src/lib/news/index.ts b/src/lib/news/index.ts new file mode 100644 index 0000000..a83dd7c --- /dev/null +++ b/src/lib/news/index.ts @@ -0,0 +1,46 @@ +import { FetchNewsOptions, RawArticle } from './types'; +import { fetchNewsApi } from './sources/newsapi'; +import { fetchGNews } from './sources/gnews'; +import { fetchWebSearch } from './sources/websearch'; +import { fetchNewsData } from './sources/newsdata'; +import { fetchBrave } from './sources/brave'; + +const sourceMap: Record Promise> = { + newsapi: fetchNewsApi, + gnews: fetchGNews, + websearch: fetchWebSearch, + newsdata: fetchNewsData, + brave: fetchBrave, +}; + +export async function fetchNews(options: FetchNewsOptions): Promise { + const sources = options.sources && options.sources.length > 0 + ? options.sources + : Object.keys(sourceMap); + + const results = await Promise.allSettled( + sources.map(async (source) => { + const fetcher = sourceMap[source.toLowerCase()]; + if (!fetcher) throw new Error(`Unknown news source: ${source}`); + return fetcher(options); + }) + ); + + const articles: RawArticle[] = []; + results.forEach((result, index) => { + if (result.status === 'fulfilled') { + articles.push(...result.value); + } else { + console.error(`Source ${sources[index]} failed:`, result.reason); + } + }); + + // Deduplicate by URL + const seen = new Set(); + return articles.filter((article) => { + const normalized = article.url.split('?')[0].toLowerCase(); + if (seen.has(normalized)) return false; + seen.add(normalized); + return true; + }); +} diff --git a/src/lib/news/sources/brave.ts b/src/lib/news/sources/brave.ts new file mode 100644 index 0000000..fd8a1e5 --- /dev/null +++ b/src/lib/news/sources/brave.ts @@ -0,0 +1,80 @@ +import { FetchNewsOptions, RawArticle } from '../types'; +import { getActiveKey } from '@/lib/api-keys'; + +function parseBraveAge(age?: string): Date | undefined { + if (!age) return undefined; + // Brave returns relative strings like "1 hour ago", "2 days ago", "Jun 16, 2026" + // We can only reliably parse the ISO-like or simple relative forms here. + // For now, fall back to 'now' if parsing fails. + try { + const match = age.match(/(\d+)\s+(minute|hour|day|week|month|year)s?\s+ago/i); + if (match) { + const value = parseInt(match[1], 10); + const unit = match[2].toLowerCase(); + const now = new Date(); + switch (unit) { + case 'minute': now.setMinutes(now.getMinutes() - value); break; + case 'hour': now.setHours(now.getHours() - value); break; + case 'day': now.setDate(now.getDate() - value); break; + case 'week': now.setDate(now.getDate() - value * 7); break; + case 'month': now.setMonth(now.getMonth() - value); break; + case 'year': now.setFullYear(now.getFullYear() - value); break; + } + return now; + } + const parsed = new Date(age); + if (!isNaN(parsed.getTime())) return parsed; + } catch { + // ignore + } + return undefined; +} + +export async function fetchBrave(options: FetchNewsOptions): Promise { + const apiKey = await getActiveKey('BRAVE'); + if (!apiKey) { + console.warn('Brave Search key not configured'); + return []; + } + + const params = new URLSearchParams({ + q: options.query, + count: String(Math.min(options.maxResults || 10, 20)), + search_lang: 'en', + text_decorations: 'false', + }); + + if (options.from) { + params.set('freshness', `dtf:${options.from.toISOString().split('T')[0]}_9999-12-31`); + } + + const res = await fetch(`https://api.search.brave.com/res/v1/news/search?${params.toString()}`, { + headers: { + 'X-Subscription-Token': apiKey, + 'Accept': 'application/json', + }, + }); + + if (!res.ok) { + const text = await res.text(); + throw new Error(`Brave search error ${res.status}: ${text}`); + } + + const data = await res.json(); + + return (data.results || []).map((item: Record) => { + const meta = item.meta as Record | undefined; + const metaUrl = meta?.url as Record | undefined; + return { + title: String(item.title || ''), + url: String(item.url || ''), + sourceName: String(metaUrl?.hostname || item.source || 'Brave Search'), + sourceApiCode: 'brave', + publishedAt: parseBraveAge(item.age ? String(item.age) : undefined), + description: item.description ? String(item.description) : undefined, + content: undefined, + imageUrl: metaUrl?.favicon ? String(metaUrl.favicon) : undefined, + raw: item, + }; + }); +} diff --git a/src/lib/news/sources/gnews.ts b/src/lib/news/sources/gnews.ts new file mode 100644 index 0000000..b0391c9 --- /dev/null +++ b/src/lib/news/sources/gnews.ts @@ -0,0 +1,40 @@ +import { FetchNewsOptions, RawArticle } from '../types'; +import { getActiveKey } from '@/lib/api-keys'; + +export async function fetchGNews(options: FetchNewsOptions): Promise { + const apiKey = await getActiveKey('GNEWS'); + if (!apiKey) { + console.warn('GNews key not configured'); + return []; + } + + const url = new URL('https://gnews.io/api/v4/search'); + url.searchParams.set('q', options.query); + url.searchParams.set('apikey', apiKey); + url.searchParams.set('max', String(Math.min(options.maxResults || 20, 100))); + url.searchParams.set('lang', 'en'); + url.searchParams.set('sortby', 'publishedAt'); + + if (options.from) url.searchParams.set('from', options.from.toISOString()); + if (options.to) url.searchParams.set('to', options.to.toISOString()); + + const res = await fetch(url.toString()); + if (!res.ok) { + const text = await res.text(); + throw new Error(`GNews error ${res.status}: ${text}`); + } + + const data = await res.json(); + + return (data.articles || []).map((article: Record) => ({ + title: String(article.title || ''), + url: String(article.url || ''), + sourceName: String((article.source as Record)?.name || 'GNews'), + sourceApiCode: 'gnews', + publishedAt: article.publishedAt ? new Date(String(article.publishedAt)) : undefined, + description: article.description ? String(article.description) : undefined, + content: article.content ? String(article.content) : undefined, + imageUrl: article.image ? String(article.image) : undefined, + raw: article, + })); +} diff --git a/src/lib/news/sources/newsapi.ts b/src/lib/news/sources/newsapi.ts new file mode 100644 index 0000000..fba073c --- /dev/null +++ b/src/lib/news/sources/newsapi.ts @@ -0,0 +1,43 @@ +import { FetchNewsOptions, RawArticle } from '../types'; +import { getActiveKey } from '@/lib/api-keys'; + +export async function fetchNewsApi(options: FetchNewsOptions): Promise { + const apiKey = await getActiveKey('NEWSAPI'); + if (!apiKey) { + console.warn('NewsAPI key not configured'); + return []; + } + + const url = new URL('https://newsapi.org/v2/everything'); + url.searchParams.set('q', options.query); + url.searchParams.set('apiKey', apiKey); + url.searchParams.set('pageSize', String(Math.min(options.maxResults || 20, 100))); + url.searchParams.set('sortBy', 'publishedAt'); + url.searchParams.set('language', 'en'); + + if (options.from) url.searchParams.set('from', options.from.toISOString()); + if (options.to) url.searchParams.set('to', options.to.toISOString()); + + const res = await fetch(url.toString()); + if (!res.ok) { + const text = await res.text(); + throw new Error(`NewsAPI error ${res.status}: ${text}`); + } + + const data = await res.json(); + if (data.status !== 'ok') { + throw new Error(`NewsAPI error: ${data.message || 'unknown'}`); + } + + return (data.articles || []).map((article: Record) => ({ + title: String(article.title || ''), + url: String(article.url || ''), + sourceName: String((article.source as Record)?.name || 'NewsAPI'), + sourceApiCode: 'newsapi', + publishedAt: article.publishedAt ? new Date(String(article.publishedAt)) : undefined, + description: article.description ? String(article.description) : undefined, + content: article.content ? String(article.content) : undefined, + imageUrl: article.urlToImage ? String(article.urlToImage) : undefined, + raw: article, + })); +} diff --git a/src/lib/news/sources/newsdata.ts b/src/lib/news/sources/newsdata.ts new file mode 100644 index 0000000..08c0b9a --- /dev/null +++ b/src/lib/news/sources/newsdata.ts @@ -0,0 +1,42 @@ +import { FetchNewsOptions, RawArticle } from '../types'; +import { getActiveKey } from '@/lib/api-keys'; + +export async function fetchNewsData(options: FetchNewsOptions): Promise { + const apiKey = await getActiveKey('NEWSDATA'); + if (!apiKey) { + console.warn('NewsData key not configured'); + return []; + } + + const url = new URL('https://newsdata.io/api/1/latest'); + url.searchParams.set('q', options.query); + url.searchParams.set('apikey', apiKey); + url.searchParams.set('size', String(Math.min(options.maxResults || 10, 50))); + url.searchParams.set('language', 'en'); + + if (options.from) url.searchParams.set('from_date', options.from.toISOString().split('T')[0]); + if (options.to) url.searchParams.set('to_date', options.to.toISOString().split('T')[0]); + + const res = await fetch(url.toString()); + if (!res.ok) { + const text = await res.text(); + throw new Error(`NewsData error ${res.status}: ${text}`); + } + + const data = await res.json(); + if (data.status !== 'success') { + throw new Error(`NewsData error: ${data.message || data.status || 'unknown'}`); + } + + return (data.results || []).map((article: Record) => ({ + title: String(article.title || ''), + url: String(article.link || ''), + sourceName: String(article.source_id || 'NewsData'), + sourceApiCode: 'newsdata', + publishedAt: article.pubDate ? new Date(String(article.pubDate)) : undefined, + description: article.description ? String(article.description) : undefined, + content: article.content ? String(article.content) : undefined, + imageUrl: article.image_url ? String(article.image_url) : undefined, + raw: article, + })); +} diff --git a/src/lib/news/sources/websearch.ts b/src/lib/news/sources/websearch.ts new file mode 100644 index 0000000..3955682 --- /dev/null +++ b/src/lib/news/sources/websearch.ts @@ -0,0 +1,42 @@ +import { FetchNewsOptions, RawArticle } from '../types'; +import { getActiveKey } from '@/lib/api-keys'; + +export async function fetchWebSearch(options: FetchNewsOptions): Promise { + const apiKey = await getActiveKey('WEBSEARCH'); + if (!apiKey) { + console.warn('Web search key not configured'); + return []; + } + + const res = await fetch('https://google.serper.dev/search', { + method: 'POST', + headers: { + 'X-API-KEY': apiKey, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + q: options.query, + num: Math.min(options.maxResults || 10, 20), + tbs: options.from ? `cdr:1,cd_min:${options.from.toISOString().split('T')[0]}` : undefined, + }), + }); + + if (!res.ok) { + const text = await res.text(); + throw new Error(`Web search error ${res.status}: ${text}`); + } + + const data = await res.json(); + + return (data.news || data.organic || []).map((item: Record) => ({ + title: String(item.title || ''), + url: String(item.link || item.url || ''), + sourceName: String(item.source || 'Web Search'), + sourceApiCode: 'websearch', + publishedAt: item.date ? new Date(String(item.date)) : undefined, + description: item.snippet ? String(item.snippet) : undefined, + content: undefined, + imageUrl: item.imageUrl ? String(item.imageUrl) : undefined, + raw: item, + })); +} diff --git a/src/lib/news/types.ts b/src/lib/news/types.ts new file mode 100644 index 0000000..46cbd89 --- /dev/null +++ b/src/lib/news/types.ts @@ -0,0 +1,19 @@ +export type RawArticle = { + title: string; + url: string; + sourceName: string; + sourceApiCode: string; + publishedAt?: Date; + description?: string; + content?: string; + imageUrl?: string; + raw: Record; +}; + +export type FetchNewsOptions = { + query: string; + sources?: string[]; + maxResults?: number; + from?: Date; + to?: Date; +};