feat: add news source adapters and AI article analysis
- Implement adapters for NewsAPI, GNews, NewsData, Serper Web Search, and Brave Search. - Add unified fetchNews aggregator with URL deduplication. - Wire OpenAI-compatible Kimi client for bias classification, ranking, summarization, and topic extraction. - Add AI analysis prompts, response parsing, and fallback logic. - Build ranker combining AI score, source credibility, and recency. - Create admin Discover page and API for manual news discovery.
This commit is contained in:
233
src/app/admin/discover/page.tsx
Normal file
233
src/app/admin/discover/page.tsx
Normal file
@@ -0,0 +1,233 @@
|
|||||||
|
'use client';
|
||||||
|
|
||||||
|
import { useEffect, useState } from 'react';
|
||||||
|
import { Bias } from '@prisma/client';
|
||||||
|
|
||||||
|
const availableSources = [
|
||||||
|
{ code: 'newsapi', name: 'NewsAPI' },
|
||||||
|
{ code: 'gnews', name: 'GNews' },
|
||||||
|
{ code: 'websearch', name: 'Web Search (Serper)' },
|
||||||
|
];
|
||||||
|
|
||||||
|
const biasColors: Record<Bias, string> = {
|
||||||
|
LEFT: 'bg-blue-100 text-blue-800 dark:bg-blue-950 dark:text-blue-300',
|
||||||
|
CENTER: 'bg-purple-100 text-purple-800 dark:bg-purple-950 dark:text-purple-300',
|
||||||
|
RIGHT: 'bg-red-100 text-red-800 dark:bg-red-950 dark:text-red-300',
|
||||||
|
};
|
||||||
|
|
||||||
|
type DiscoveredArticle = {
|
||||||
|
success: boolean;
|
||||||
|
article?: {
|
||||||
|
id: string;
|
||||||
|
title: string;
|
||||||
|
summary: string;
|
||||||
|
url: string;
|
||||||
|
sourceName: string;
|
||||||
|
primaryBias: Bias;
|
||||||
|
politicalTags: string[];
|
||||||
|
rankScore: number;
|
||||||
|
credibilityScore: number;
|
||||||
|
topics: string[];
|
||||||
|
status: string;
|
||||||
|
};
|
||||||
|
error?: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export default function DiscoverPage() {
|
||||||
|
const [query, setQuery] = useState('');
|
||||||
|
const [selectedSources, setSelectedSources] = useState<string[]>(['newsapi', 'gnews']);
|
||||||
|
const [maxResults, setMaxResults] = useState(10);
|
||||||
|
const [autoPublish, setAutoPublish] = useState(false);
|
||||||
|
const [loading, setLoading] = useState(false);
|
||||||
|
const [results, setResults] = useState<DiscoveredArticle[]>([]);
|
||||||
|
const [stats, setStats] = useState<{ found: number; processed: number; failed: number } | null>(null);
|
||||||
|
|
||||||
|
const toggleSource = (code: string) => {
|
||||||
|
setSelectedSources((prev) =>
|
||||||
|
prev.includes(code) ? prev.filter((c) => c !== code) : [...prev, code]
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
const handleDiscover = async (e: React.FormEvent) => {
|
||||||
|
e.preventDefault();
|
||||||
|
if (selectedSources.length === 0) {
|
||||||
|
alert('Select at least one source');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
setLoading(true);
|
||||||
|
setResults([]);
|
||||||
|
setStats(null);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const res = await fetch('/api/admin/discover', {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
body: JSON.stringify({
|
||||||
|
query,
|
||||||
|
sources: selectedSources,
|
||||||
|
maxResults,
|
||||||
|
autoPublish,
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
|
||||||
|
const data = await res.json();
|
||||||
|
if (!res.ok) {
|
||||||
|
alert(data.error || 'Discovery failed');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
setResults(data.results || []);
|
||||||
|
setStats({ found: data.found, processed: data.processed, failed: data.failed });
|
||||||
|
} finally {
|
||||||
|
setLoading(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div>
|
||||||
|
<h1 className="text-2xl font-bold mb-6">Discover News</h1>
|
||||||
|
|
||||||
|
<div className="bg-white dark:bg-neutral-900 rounded-xl shadow-sm border dark:border-neutral-800 p-6 mb-6">
|
||||||
|
<form onSubmit={handleDiscover} className="space-y-4">
|
||||||
|
<div>
|
||||||
|
<label className="block text-sm font-medium mb-1">Search Query</label>
|
||||||
|
<input
|
||||||
|
type="text"
|
||||||
|
value={query}
|
||||||
|
onChange={(e) => setQuery(e.target.value)}
|
||||||
|
className="w-full px-3 py-2 border rounded-lg dark:bg-neutral-800 dark:border-neutral-700"
|
||||||
|
placeholder="e.g. US politics, climate policy, Supreme Court"
|
||||||
|
required
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div>
|
||||||
|
<label className="block text-sm font-medium mb-2">Sources</label>
|
||||||
|
<div className="flex flex-wrap gap-3">
|
||||||
|
{availableSources.map((s) => (
|
||||||
|
<label
|
||||||
|
key={s.code}
|
||||||
|
className={`flex items-center gap-2 px-3 py-2 rounded-lg border cursor-pointer ${
|
||||||
|
selectedSources.includes(s.code)
|
||||||
|
? 'bg-blue-50 border-blue-300 dark:bg-blue-950 dark:border-blue-700'
|
||||||
|
: 'bg-white dark:bg-neutral-900 border-neutral-200 dark:border-neutral-700'
|
||||||
|
}`}
|
||||||
|
>
|
||||||
|
<input
|
||||||
|
type="checkbox"
|
||||||
|
checked={selectedSources.includes(s.code)}
|
||||||
|
onChange={() => toggleSource(s.code)}
|
||||||
|
/>
|
||||||
|
<span className="text-sm">{s.name}</span>
|
||||||
|
</label>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="grid grid-cols-1 md:grid-cols-3 gap-4 items-end">
|
||||||
|
<div>
|
||||||
|
<label className="block text-sm font-medium mb-1">Max Results per Source</label>
|
||||||
|
<input
|
||||||
|
type="number"
|
||||||
|
min={1}
|
||||||
|
max={50}
|
||||||
|
value={maxResults}
|
||||||
|
onChange={(e) => setMaxResults(Number(e.target.value))}
|
||||||
|
className="w-full px-3 py-2 border rounded-lg dark:bg-neutral-800 dark:border-neutral-700"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<div className="flex items-center gap-2">
|
||||||
|
<input
|
||||||
|
id="autoPublish"
|
||||||
|
type="checkbox"
|
||||||
|
checked={autoPublish}
|
||||||
|
onChange={(e) => setAutoPublish(e.target.checked)}
|
||||||
|
/>
|
||||||
|
<label htmlFor="autoPublish" className="text-sm font-medium">
|
||||||
|
Auto-publish discovered articles
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
|
<div className="text-right">
|
||||||
|
<button
|
||||||
|
type="submit"
|
||||||
|
disabled={loading}
|
||||||
|
className="px-4 py-2 bg-blue-600 hover:bg-blue-700 text-white rounded-lg font-medium disabled:opacity-50"
|
||||||
|
>
|
||||||
|
{loading ? 'Discovering...' : 'Discover News'}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{stats && (
|
||||||
|
<div className="mb-4 text-sm text-neutral-600 dark:text-neutral-400">
|
||||||
|
Found {stats.found} raw articles. Processed {stats.processed}, failed {stats.failed}.
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
<div className="space-y-4">
|
||||||
|
{results.map((result, idx) => {
|
||||||
|
if (!result.success || !result.article) {
|
||||||
|
return (
|
||||||
|
<div
|
||||||
|
key={idx}
|
||||||
|
className="bg-red-50 dark:bg-red-950 border border-red-200 dark:border-red-900 rounded-xl p-4 text-sm text-red-700 dark:text-red-300"
|
||||||
|
>
|
||||||
|
Failed: {result.error}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const a = result.article;
|
||||||
|
return (
|
||||||
|
<div
|
||||||
|
key={a.id}
|
||||||
|
className="bg-white dark:bg-neutral-900 rounded-xl shadow-sm border dark:border-neutral-800 p-5"
|
||||||
|
>
|
||||||
|
<div className="flex flex-wrap items-start justify-between gap-3 mb-2">
|
||||||
|
<h3 className="text-lg font-semibold">
|
||||||
|
<a href={a.url} target="_blank" rel="noopener noreferrer" className="hover:underline">
|
||||||
|
{a.title}
|
||||||
|
</a>
|
||||||
|
</h3>
|
||||||
|
<span
|
||||||
|
className={`px-2 py-1 rounded-full text-xs font-medium ${biasColors[a.primaryBias]}`}
|
||||||
|
>
|
||||||
|
{a.primaryBias}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
<p className="text-sm text-neutral-600 dark:text-neutral-400 mb-3 line-clamp-3">
|
||||||
|
{a.summary}
|
||||||
|
</p>
|
||||||
|
<div className="flex flex-wrap gap-2 mb-3">
|
||||||
|
{a.politicalTags.map((tag) => (
|
||||||
|
<span
|
||||||
|
key={tag}
|
||||||
|
className="px-2 py-1 rounded-md text-xs bg-neutral-100 dark:bg-neutral-800"
|
||||||
|
>
|
||||||
|
{tag}
|
||||||
|
</span>
|
||||||
|
))}
|
||||||
|
{a.topics.slice(0, 5).map((topic) => (
|
||||||
|
<span
|
||||||
|
key={topic}
|
||||||
|
className="px-2 py-1 rounded-md text-xs bg-blue-50 dark:bg-blue-950 text-blue-700 dark:text-blue-300"
|
||||||
|
>
|
||||||
|
{topic}
|
||||||
|
</span>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
<div className="flex flex-wrap gap-4 text-xs text-neutral-500 dark:text-neutral-400">
|
||||||
|
<span>Source: {a.sourceName}</span>
|
||||||
|
<span>Rank: {a.rankScore}/10</span>
|
||||||
|
<span>Credibility: {a.credibilityScore}/10</span>
|
||||||
|
<span>Status: {a.status}</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
})}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
58
src/app/api/admin/discover/route.ts
Normal file
58
src/app/api/admin/discover/route.ts
Normal file
@@ -0,0 +1,58 @@
|
|||||||
|
import { NextResponse } from 'next/server';
|
||||||
|
import { fetchNews } from '@/lib/news';
|
||||||
|
import { processArticle } from '@/lib/ai/processor';
|
||||||
|
import { saveProcessedArticle } from '@/lib/articles';
|
||||||
|
|
||||||
|
export async function POST(request: Request) {
|
||||||
|
try {
|
||||||
|
const body = await request.json();
|
||||||
|
const { query, sources, maxResults, autoPublish } = body;
|
||||||
|
|
||||||
|
if (!query || typeof query !== 'string') {
|
||||||
|
return NextResponse.json({ error: 'Query is required' }, { status: 400 });
|
||||||
|
}
|
||||||
|
|
||||||
|
const rawArticles = await fetchNews({
|
||||||
|
query,
|
||||||
|
sources: sources || undefined,
|
||||||
|
maxResults: maxResults || 20,
|
||||||
|
});
|
||||||
|
|
||||||
|
const processed = await Promise.allSettled(
|
||||||
|
rawArticles.map(async (raw) => {
|
||||||
|
const analyzed = await processArticle(raw);
|
||||||
|
const status = autoPublish ? 'PUBLISHED' : 'DRAFT';
|
||||||
|
const saved = await saveProcessedArticle(analyzed, status);
|
||||||
|
return {
|
||||||
|
...analyzed,
|
||||||
|
id: saved.id,
|
||||||
|
status,
|
||||||
|
politicalTags: analyzed.politicalTags,
|
||||||
|
topics: analyzed.topics,
|
||||||
|
};
|
||||||
|
})
|
||||||
|
);
|
||||||
|
|
||||||
|
const results = processed.map((result) =>
|
||||||
|
result.status === 'fulfilled'
|
||||||
|
? { success: true, article: result.value }
|
||||||
|
: { success: false, error: String(result.reason) }
|
||||||
|
);
|
||||||
|
|
||||||
|
const succeeded = results.filter((r) => r.success);
|
||||||
|
const failed = results.filter((r) => !r.success);
|
||||||
|
|
||||||
|
return NextResponse.json({
|
||||||
|
found: rawArticles.length,
|
||||||
|
processed: succeeded.length,
|
||||||
|
failed: failed.length,
|
||||||
|
results,
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Discover error:', error);
|
||||||
|
return NextResponse.json(
|
||||||
|
{ error: String(error) },
|
||||||
|
{ status: 500 }
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
17
src/lib/ai/client.ts
Normal file
17
src/lib/ai/client.ts
Normal file
@@ -0,0 +1,17 @@
|
|||||||
|
import OpenAI from 'openai';
|
||||||
|
import { getActiveKey } from '../api-keys';
|
||||||
|
|
||||||
|
export async function getAiClient(): Promise<OpenAI> {
|
||||||
|
const apiKey = await getActiveKey('KIMI');
|
||||||
|
const baseURL = process.env.KIMI_API_URL || 'https://api.moonshot.ai/v1';
|
||||||
|
|
||||||
|
if (!apiKey) {
|
||||||
|
throw new Error('Kimi API key not configured. Add one in Admin > API Keys.');
|
||||||
|
}
|
||||||
|
|
||||||
|
return new OpenAI({ apiKey, baseURL, timeout: 60_000, maxRetries: 1 });
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getAiModel(): string {
|
||||||
|
return process.env.KIMI_MODEL || 'kimi-k2.5';
|
||||||
|
}
|
||||||
95
src/lib/ai/processor.ts
Normal file
95
src/lib/ai/processor.ts
Normal file
@@ -0,0 +1,95 @@
|
|||||||
|
import { RawArticle } from '../news/types';
|
||||||
|
import { getAiClient, getAiModel } from './client';
|
||||||
|
import { buildArticleAnalysisPrompt } from './prompts';
|
||||||
|
import { Bias } from '@prisma/client';
|
||||||
|
|
||||||
|
export type ProcessedArticle = {
|
||||||
|
title: string;
|
||||||
|
summary: string;
|
||||||
|
url: string;
|
||||||
|
sourceName: string;
|
||||||
|
sourceApiCode: string;
|
||||||
|
publishedAt?: Date;
|
||||||
|
imageUrl?: string;
|
||||||
|
primaryBias: Bias;
|
||||||
|
biasConfidence: number;
|
||||||
|
politicalTags: string[];
|
||||||
|
rankScore: number;
|
||||||
|
credibilityScore: number;
|
||||||
|
topics: string[];
|
||||||
|
aiExplanation: string;
|
||||||
|
raw: Record<string, unknown>;
|
||||||
|
};
|
||||||
|
|
||||||
|
function safeParseBias(value: string): Bias {
|
||||||
|
const upper = String(value).toUpperCase();
|
||||||
|
if (upper === 'LEFT' || upper === 'CENTER' || upper === 'RIGHT') {
|
||||||
|
return upper as Bias;
|
||||||
|
}
|
||||||
|
return 'CENTER';
|
||||||
|
}
|
||||||
|
|
||||||
|
function safeParseNumber(value: unknown, fallback: number, min = 0, max = 10): number {
|
||||||
|
const num = Number(value);
|
||||||
|
if (Number.isNaN(num)) return fallback;
|
||||||
|
return Math.max(min, Math.min(max, num));
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function processArticle(
|
||||||
|
raw: RawArticle,
|
||||||
|
options?: { model?: string }
|
||||||
|
): Promise<ProcessedArticle> {
|
||||||
|
const client = await getAiClient();
|
||||||
|
const prompt = buildArticleAnalysisPrompt({
|
||||||
|
title: raw.title,
|
||||||
|
sourceName: raw.sourceName,
|
||||||
|
description: raw.description,
|
||||||
|
content: raw.content,
|
||||||
|
url: raw.url,
|
||||||
|
});
|
||||||
|
|
||||||
|
const res = await client.chat.completions.create({
|
||||||
|
model: options?.model || getAiModel(),
|
||||||
|
messages: [{ role: 'user', content: prompt }],
|
||||||
|
});
|
||||||
|
|
||||||
|
const content = res.choices[0]?.message?.content?.trim() || '{}';
|
||||||
|
let parsed: Record<string, unknown> = {};
|
||||||
|
|
||||||
|
try {
|
||||||
|
const cleaned = content.replace(/^```json\s*|\s*```$/g, '');
|
||||||
|
parsed = JSON.parse(cleaned);
|
||||||
|
} catch (err) {
|
||||||
|
console.error('Failed to parse AI response:', content, err);
|
||||||
|
parsed = {};
|
||||||
|
}
|
||||||
|
|
||||||
|
const politicalTags = Array.isArray(parsed.politicalTags)
|
||||||
|
? parsed.politicalTags.map(String).filter((t) => ['liberal', 'moderate', 'conservative'].includes(t.toLowerCase()))
|
||||||
|
: [];
|
||||||
|
|
||||||
|
if (politicalTags.length === 0) {
|
||||||
|
const bias = safeParseBias(String(parsed.primaryBias));
|
||||||
|
if (bias === 'LEFT') politicalTags.push('liberal');
|
||||||
|
if (bias === 'CENTER') politicalTags.push('moderate');
|
||||||
|
if (bias === 'RIGHT') politicalTags.push('conservative');
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
title: raw.title,
|
||||||
|
summary: String(parsed.summary || raw.description || ''),
|
||||||
|
url: raw.url,
|
||||||
|
sourceName: raw.sourceName,
|
||||||
|
sourceApiCode: raw.sourceApiCode,
|
||||||
|
publishedAt: raw.publishedAt,
|
||||||
|
imageUrl: raw.imageUrl,
|
||||||
|
primaryBias: safeParseBias(parsed.primaryBias as string),
|
||||||
|
biasConfidence: safeParseNumber(parsed.biasConfidence, 0.5, 0, 1),
|
||||||
|
politicalTags,
|
||||||
|
rankScore: safeParseNumber(parsed.rankScore, 5, 1, 10),
|
||||||
|
credibilityScore: safeParseNumber(parsed.credibilityScore, 5, 1, 10),
|
||||||
|
topics: Array.isArray(parsed.topics) ? parsed.topics.map(String) : [],
|
||||||
|
aiExplanation: String(parsed.explanation || ''),
|
||||||
|
raw: raw.raw,
|
||||||
|
};
|
||||||
|
}
|
||||||
55
src/lib/ai/prompts.ts
Normal file
55
src/lib/ai/prompts.ts
Normal file
@@ -0,0 +1,55 @@
|
|||||||
|
export function buildArticleAnalysisPrompt(article: {
|
||||||
|
title: string;
|
||||||
|
sourceName: string;
|
||||||
|
description?: string;
|
||||||
|
content?: string;
|
||||||
|
url: string;
|
||||||
|
}): string {
|
||||||
|
const text = [article.title, article.description, article.content]
|
||||||
|
.filter(Boolean)
|
||||||
|
.join('\n\n');
|
||||||
|
|
||||||
|
return `Analyze the following news article and return ONLY a JSON object with no markdown formatting.
|
||||||
|
|
||||||
|
Article URL: ${article.url}
|
||||||
|
Source: ${article.sourceName}
|
||||||
|
Text:
|
||||||
|
"""
|
||||||
|
${text.slice(0, 12000)}
|
||||||
|
"""
|
||||||
|
|
||||||
|
Return JSON in this exact shape:
|
||||||
|
{
|
||||||
|
"summary": "A concise 2-3 sentence summary of the article.",
|
||||||
|
"primaryBias": "LEFT" | "CENTER" | "RIGHT",
|
||||||
|
"biasConfidence": 0.0 to 1.0,
|
||||||
|
"politicalTags": ["liberal" | "moderate" | "conservative"],
|
||||||
|
"rankScore": 1 to 10,
|
||||||
|
"credibilityScore": 1 to 10,
|
||||||
|
"topics": ["topic1", "topic2"],
|
||||||
|
"explanation": "One sentence explaining the bias classification."
|
||||||
|
}
|
||||||
|
|
||||||
|
Definitions:
|
||||||
|
- LEFT: favors progressive/liberal policy, Democrats, or left-wing framing.
|
||||||
|
- CENTER: neutral, balanced, wire-service, or mainstream non-partisan framing.
|
||||||
|
- RIGHT: favors conservative/traditional policy, Republicans, or right-wing framing.
|
||||||
|
politicalTags should include one or more of: liberal, moderate, conservative.
|
||||||
|
rankScore reflects how important/significant the story is (1 = trivial, 10 = major).
|
||||||
|
credibilityScore reflects source reliability (1 = unreliable, 10 = highly credible).`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildNewsSearchPrompt(query: string): string {
|
||||||
|
return `Find recent news articles about "${query}". Return ONLY a JSON array of objects with no markdown formatting.
|
||||||
|
|
||||||
|
Each object must have:
|
||||||
|
{
|
||||||
|
"title": "Article title",
|
||||||
|
"url": "Article URL",
|
||||||
|
"sourceName": "Source name",
|
||||||
|
"publishedAt": "ISO date string or null",
|
||||||
|
"description": "Brief description or snippet"
|
||||||
|
}
|
||||||
|
|
||||||
|
Return at most 10 articles. If you cannot browse the live web, return an empty array [].`;
|
||||||
|
}
|
||||||
21
src/lib/ai/ranker.ts
Normal file
21
src/lib/ai/ranker.ts
Normal file
@@ -0,0 +1,21 @@
|
|||||||
|
export function computeFinalRankScore({
|
||||||
|
aiRankScore,
|
||||||
|
credibilityScore,
|
||||||
|
publishedAt,
|
||||||
|
}: {
|
||||||
|
aiRankScore: number;
|
||||||
|
credibilityScore: number;
|
||||||
|
publishedAt?: Date;
|
||||||
|
}): number {
|
||||||
|
// Base score: 60% AI rank, 40% credibility
|
||||||
|
let score = aiRankScore * 0.6 + credibilityScore * 0.4;
|
||||||
|
|
||||||
|
// Recency boost: up to +1.5 for articles published in last 24h, decaying over 7 days
|
||||||
|
if (publishedAt) {
|
||||||
|
const ageHours = (Date.now() - publishedAt.getTime()) / (1000 * 60 * 60);
|
||||||
|
const recencyBoost = Math.max(0, 1.5 * (1 - ageHours / (24 * 7)));
|
||||||
|
score += recencyBoost;
|
||||||
|
}
|
||||||
|
|
||||||
|
return Math.min(10, Math.max(1, Number(score.toFixed(2))));
|
||||||
|
}
|
||||||
106
src/lib/articles.ts
Normal file
106
src/lib/articles.ts
Normal file
@@ -0,0 +1,106 @@
|
|||||||
|
import { prisma } from './db';
|
||||||
|
import { ProcessedArticle } from './ai/processor';
|
||||||
|
import { computeFinalRankScore } from './ai/ranker';
|
||||||
|
import { ArticleStatus, Bias, Prisma } from '@prisma/client';
|
||||||
|
|
||||||
|
export async function findOrCreateSource(apiCode: string, name: string) {
|
||||||
|
return prisma.source.upsert({
|
||||||
|
where: { apiCode },
|
||||||
|
update: {},
|
||||||
|
create: { apiCode, name },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function saveProcessedArticle(
|
||||||
|
processed: ProcessedArticle,
|
||||||
|
status: ArticleStatus = 'DRAFT'
|
||||||
|
) {
|
||||||
|
const source = await findOrCreateSource(processed.sourceApiCode, processed.sourceName);
|
||||||
|
|
||||||
|
const existing = await prisma.article.findUnique({
|
||||||
|
where: { url: processed.url },
|
||||||
|
});
|
||||||
|
|
||||||
|
const finalRank = computeFinalRankScore({
|
||||||
|
aiRankScore: processed.rankScore,
|
||||||
|
credibilityScore: processed.credibilityScore,
|
||||||
|
publishedAt: processed.publishedAt,
|
||||||
|
});
|
||||||
|
|
||||||
|
const data: Prisma.ArticleUncheckedCreateInput = {
|
||||||
|
title: processed.title,
|
||||||
|
summary: processed.summary,
|
||||||
|
url: processed.url,
|
||||||
|
sourceId: source.id,
|
||||||
|
imageUrl: processed.imageUrl,
|
||||||
|
publishedAt: processed.publishedAt,
|
||||||
|
primaryBias: processed.primaryBias as Bias,
|
||||||
|
biasConfidence: processed.biasConfidence,
|
||||||
|
politicalTags: JSON.stringify(processed.politicalTags),
|
||||||
|
rankScore: finalRank,
|
||||||
|
credibilityScore: processed.credibilityScore,
|
||||||
|
status,
|
||||||
|
topics: JSON.stringify(processed.topics),
|
||||||
|
rawData: JSON.stringify(processed.raw),
|
||||||
|
aiExplanation: processed.aiExplanation,
|
||||||
|
};
|
||||||
|
|
||||||
|
if (existing) {
|
||||||
|
return prisma.article.update({
|
||||||
|
where: { id: existing.id },
|
||||||
|
data: {
|
||||||
|
...data,
|
||||||
|
id: undefined,
|
||||||
|
url: undefined,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
return prisma.article.create({ data });
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function getPublishedArticles(filters?: {
|
||||||
|
bias?: Bias;
|
||||||
|
query?: string;
|
||||||
|
sourceId?: string;
|
||||||
|
topic?: string;
|
||||||
|
from?: Date;
|
||||||
|
to?: Date;
|
||||||
|
limit?: number;
|
||||||
|
offset?: number;
|
||||||
|
}) {
|
||||||
|
const where: Prisma.ArticleWhereInput = { status: 'PUBLISHED' };
|
||||||
|
|
||||||
|
if (filters?.bias) where.primaryBias = filters.bias;
|
||||||
|
if (filters?.sourceId) where.sourceId = filters.sourceId;
|
||||||
|
if (filters?.topic) {
|
||||||
|
where.topics = { contains: `"${filters.topic}"` };
|
||||||
|
}
|
||||||
|
if (filters?.from || filters?.to) {
|
||||||
|
where.publishedAt = {};
|
||||||
|
if (filters.from) where.publishedAt.gte = filters.from;
|
||||||
|
if (filters.to) where.publishedAt.lte = filters.to;
|
||||||
|
}
|
||||||
|
if (filters?.query) {
|
||||||
|
where.OR = [
|
||||||
|
{ title: { contains: filters.query } },
|
||||||
|
{ summary: { contains: filters.query } },
|
||||||
|
{ topics: { contains: filters.query } },
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
return prisma.article.findMany({
|
||||||
|
where,
|
||||||
|
orderBy: [{ rankScore: 'desc' }, { publishedAt: 'desc' }],
|
||||||
|
take: filters?.limit ?? 50,
|
||||||
|
skip: filters?.offset ?? 0,
|
||||||
|
include: { source: true },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function getArticleById(id: string) {
|
||||||
|
return prisma.article.findUnique({
|
||||||
|
where: { id },
|
||||||
|
include: { source: true },
|
||||||
|
});
|
||||||
|
}
|
||||||
46
src/lib/news/index.ts
Normal file
46
src/lib/news/index.ts
Normal file
@@ -0,0 +1,46 @@
|
|||||||
|
import { FetchNewsOptions, RawArticle } from './types';
|
||||||
|
import { fetchNewsApi } from './sources/newsapi';
|
||||||
|
import { fetchGNews } from './sources/gnews';
|
||||||
|
import { fetchWebSearch } from './sources/websearch';
|
||||||
|
import { fetchNewsData } from './sources/newsdata';
|
||||||
|
import { fetchBrave } from './sources/brave';
|
||||||
|
|
||||||
|
const sourceMap: Record<string, (opts: FetchNewsOptions) => Promise<RawArticle[]>> = {
|
||||||
|
newsapi: fetchNewsApi,
|
||||||
|
gnews: fetchGNews,
|
||||||
|
websearch: fetchWebSearch,
|
||||||
|
newsdata: fetchNewsData,
|
||||||
|
brave: fetchBrave,
|
||||||
|
};
|
||||||
|
|
||||||
|
export async function fetchNews(options: FetchNewsOptions): Promise<RawArticle[]> {
|
||||||
|
const sources = options.sources && options.sources.length > 0
|
||||||
|
? options.sources
|
||||||
|
: Object.keys(sourceMap);
|
||||||
|
|
||||||
|
const results = await Promise.allSettled(
|
||||||
|
sources.map(async (source) => {
|
||||||
|
const fetcher = sourceMap[source.toLowerCase()];
|
||||||
|
if (!fetcher) throw new Error(`Unknown news source: ${source}`);
|
||||||
|
return fetcher(options);
|
||||||
|
})
|
||||||
|
);
|
||||||
|
|
||||||
|
const articles: RawArticle[] = [];
|
||||||
|
results.forEach((result, index) => {
|
||||||
|
if (result.status === 'fulfilled') {
|
||||||
|
articles.push(...result.value);
|
||||||
|
} else {
|
||||||
|
console.error(`Source ${sources[index]} failed:`, result.reason);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Deduplicate by URL
|
||||||
|
const seen = new Set<string>();
|
||||||
|
return articles.filter((article) => {
|
||||||
|
const normalized = article.url.split('?')[0].toLowerCase();
|
||||||
|
if (seen.has(normalized)) return false;
|
||||||
|
seen.add(normalized);
|
||||||
|
return true;
|
||||||
|
});
|
||||||
|
}
|
||||||
80
src/lib/news/sources/brave.ts
Normal file
80
src/lib/news/sources/brave.ts
Normal file
@@ -0,0 +1,80 @@
|
|||||||
|
import { FetchNewsOptions, RawArticle } from '../types';
|
||||||
|
import { getActiveKey } from '@/lib/api-keys';
|
||||||
|
|
||||||
|
function parseBraveAge(age?: string): Date | undefined {
|
||||||
|
if (!age) return undefined;
|
||||||
|
// Brave returns relative strings like "1 hour ago", "2 days ago", "Jun 16, 2026"
|
||||||
|
// We can only reliably parse the ISO-like or simple relative forms here.
|
||||||
|
// For now, fall back to 'now' if parsing fails.
|
||||||
|
try {
|
||||||
|
const match = age.match(/(\d+)\s+(minute|hour|day|week|month|year)s?\s+ago/i);
|
||||||
|
if (match) {
|
||||||
|
const value = parseInt(match[1], 10);
|
||||||
|
const unit = match[2].toLowerCase();
|
||||||
|
const now = new Date();
|
||||||
|
switch (unit) {
|
||||||
|
case 'minute': now.setMinutes(now.getMinutes() - value); break;
|
||||||
|
case 'hour': now.setHours(now.getHours() - value); break;
|
||||||
|
case 'day': now.setDate(now.getDate() - value); break;
|
||||||
|
case 'week': now.setDate(now.getDate() - value * 7); break;
|
||||||
|
case 'month': now.setMonth(now.getMonth() - value); break;
|
||||||
|
case 'year': now.setFullYear(now.getFullYear() - value); break;
|
||||||
|
}
|
||||||
|
return now;
|
||||||
|
}
|
||||||
|
const parsed = new Date(age);
|
||||||
|
if (!isNaN(parsed.getTime())) return parsed;
|
||||||
|
} catch {
|
||||||
|
// ignore
|
||||||
|
}
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function fetchBrave(options: FetchNewsOptions): Promise<RawArticle[]> {
|
||||||
|
const apiKey = await getActiveKey('BRAVE');
|
||||||
|
if (!apiKey) {
|
||||||
|
console.warn('Brave Search key not configured');
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
const params = new URLSearchParams({
|
||||||
|
q: options.query,
|
||||||
|
count: String(Math.min(options.maxResults || 10, 20)),
|
||||||
|
search_lang: 'en',
|
||||||
|
text_decorations: 'false',
|
||||||
|
});
|
||||||
|
|
||||||
|
if (options.from) {
|
||||||
|
params.set('freshness', `dtf:${options.from.toISOString().split('T')[0]}_9999-12-31`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const res = await fetch(`https://api.search.brave.com/res/v1/news/search?${params.toString()}`, {
|
||||||
|
headers: {
|
||||||
|
'X-Subscription-Token': apiKey,
|
||||||
|
'Accept': 'application/json',
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!res.ok) {
|
||||||
|
const text = await res.text();
|
||||||
|
throw new Error(`Brave search error ${res.status}: ${text}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const data = await res.json();
|
||||||
|
|
||||||
|
return (data.results || []).map((item: Record<string, unknown>) => {
|
||||||
|
const meta = item.meta as Record<string, unknown> | undefined;
|
||||||
|
const metaUrl = meta?.url as Record<string, unknown> | undefined;
|
||||||
|
return {
|
||||||
|
title: String(item.title || ''),
|
||||||
|
url: String(item.url || ''),
|
||||||
|
sourceName: String(metaUrl?.hostname || item.source || 'Brave Search'),
|
||||||
|
sourceApiCode: 'brave',
|
||||||
|
publishedAt: parseBraveAge(item.age ? String(item.age) : undefined),
|
||||||
|
description: item.description ? String(item.description) : undefined,
|
||||||
|
content: undefined,
|
||||||
|
imageUrl: metaUrl?.favicon ? String(metaUrl.favicon) : undefined,
|
||||||
|
raw: item,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
}
|
||||||
40
src/lib/news/sources/gnews.ts
Normal file
40
src/lib/news/sources/gnews.ts
Normal file
@@ -0,0 +1,40 @@
|
|||||||
|
import { FetchNewsOptions, RawArticle } from '../types';
|
||||||
|
import { getActiveKey } from '@/lib/api-keys';
|
||||||
|
|
||||||
|
export async function fetchGNews(options: FetchNewsOptions): Promise<RawArticle[]> {
|
||||||
|
const apiKey = await getActiveKey('GNEWS');
|
||||||
|
if (!apiKey) {
|
||||||
|
console.warn('GNews key not configured');
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
const url = new URL('https://gnews.io/api/v4/search');
|
||||||
|
url.searchParams.set('q', options.query);
|
||||||
|
url.searchParams.set('apikey', apiKey);
|
||||||
|
url.searchParams.set('max', String(Math.min(options.maxResults || 20, 100)));
|
||||||
|
url.searchParams.set('lang', 'en');
|
||||||
|
url.searchParams.set('sortby', 'publishedAt');
|
||||||
|
|
||||||
|
if (options.from) url.searchParams.set('from', options.from.toISOString());
|
||||||
|
if (options.to) url.searchParams.set('to', options.to.toISOString());
|
||||||
|
|
||||||
|
const res = await fetch(url.toString());
|
||||||
|
if (!res.ok) {
|
||||||
|
const text = await res.text();
|
||||||
|
throw new Error(`GNews error ${res.status}: ${text}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const data = await res.json();
|
||||||
|
|
||||||
|
return (data.articles || []).map((article: Record<string, unknown>) => ({
|
||||||
|
title: String(article.title || ''),
|
||||||
|
url: String(article.url || ''),
|
||||||
|
sourceName: String((article.source as Record<string, unknown>)?.name || 'GNews'),
|
||||||
|
sourceApiCode: 'gnews',
|
||||||
|
publishedAt: article.publishedAt ? new Date(String(article.publishedAt)) : undefined,
|
||||||
|
description: article.description ? String(article.description) : undefined,
|
||||||
|
content: article.content ? String(article.content) : undefined,
|
||||||
|
imageUrl: article.image ? String(article.image) : undefined,
|
||||||
|
raw: article,
|
||||||
|
}));
|
||||||
|
}
|
||||||
43
src/lib/news/sources/newsapi.ts
Normal file
43
src/lib/news/sources/newsapi.ts
Normal file
@@ -0,0 +1,43 @@
|
|||||||
|
import { FetchNewsOptions, RawArticle } from '../types';
|
||||||
|
import { getActiveKey } from '@/lib/api-keys';
|
||||||
|
|
||||||
|
export async function fetchNewsApi(options: FetchNewsOptions): Promise<RawArticle[]> {
|
||||||
|
const apiKey = await getActiveKey('NEWSAPI');
|
||||||
|
if (!apiKey) {
|
||||||
|
console.warn('NewsAPI key not configured');
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
const url = new URL('https://newsapi.org/v2/everything');
|
||||||
|
url.searchParams.set('q', options.query);
|
||||||
|
url.searchParams.set('apiKey', apiKey);
|
||||||
|
url.searchParams.set('pageSize', String(Math.min(options.maxResults || 20, 100)));
|
||||||
|
url.searchParams.set('sortBy', 'publishedAt');
|
||||||
|
url.searchParams.set('language', 'en');
|
||||||
|
|
||||||
|
if (options.from) url.searchParams.set('from', options.from.toISOString());
|
||||||
|
if (options.to) url.searchParams.set('to', options.to.toISOString());
|
||||||
|
|
||||||
|
const res = await fetch(url.toString());
|
||||||
|
if (!res.ok) {
|
||||||
|
const text = await res.text();
|
||||||
|
throw new Error(`NewsAPI error ${res.status}: ${text}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const data = await res.json();
|
||||||
|
if (data.status !== 'ok') {
|
||||||
|
throw new Error(`NewsAPI error: ${data.message || 'unknown'}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
return (data.articles || []).map((article: Record<string, unknown>) => ({
|
||||||
|
title: String(article.title || ''),
|
||||||
|
url: String(article.url || ''),
|
||||||
|
sourceName: String((article.source as Record<string, unknown>)?.name || 'NewsAPI'),
|
||||||
|
sourceApiCode: 'newsapi',
|
||||||
|
publishedAt: article.publishedAt ? new Date(String(article.publishedAt)) : undefined,
|
||||||
|
description: article.description ? String(article.description) : undefined,
|
||||||
|
content: article.content ? String(article.content) : undefined,
|
||||||
|
imageUrl: article.urlToImage ? String(article.urlToImage) : undefined,
|
||||||
|
raw: article,
|
||||||
|
}));
|
||||||
|
}
|
||||||
42
src/lib/news/sources/newsdata.ts
Normal file
42
src/lib/news/sources/newsdata.ts
Normal file
@@ -0,0 +1,42 @@
|
|||||||
|
import { FetchNewsOptions, RawArticle } from '../types';
|
||||||
|
import { getActiveKey } from '@/lib/api-keys';
|
||||||
|
|
||||||
|
export async function fetchNewsData(options: FetchNewsOptions): Promise<RawArticle[]> {
|
||||||
|
const apiKey = await getActiveKey('NEWSDATA');
|
||||||
|
if (!apiKey) {
|
||||||
|
console.warn('NewsData key not configured');
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
const url = new URL('https://newsdata.io/api/1/latest');
|
||||||
|
url.searchParams.set('q', options.query);
|
||||||
|
url.searchParams.set('apikey', apiKey);
|
||||||
|
url.searchParams.set('size', String(Math.min(options.maxResults || 10, 50)));
|
||||||
|
url.searchParams.set('language', 'en');
|
||||||
|
|
||||||
|
if (options.from) url.searchParams.set('from_date', options.from.toISOString().split('T')[0]);
|
||||||
|
if (options.to) url.searchParams.set('to_date', options.to.toISOString().split('T')[0]);
|
||||||
|
|
||||||
|
const res = await fetch(url.toString());
|
||||||
|
if (!res.ok) {
|
||||||
|
const text = await res.text();
|
||||||
|
throw new Error(`NewsData error ${res.status}: ${text}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const data = await res.json();
|
||||||
|
if (data.status !== 'success') {
|
||||||
|
throw new Error(`NewsData error: ${data.message || data.status || 'unknown'}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
return (data.results || []).map((article: Record<string, unknown>) => ({
|
||||||
|
title: String(article.title || ''),
|
||||||
|
url: String(article.link || ''),
|
||||||
|
sourceName: String(article.source_id || 'NewsData'),
|
||||||
|
sourceApiCode: 'newsdata',
|
||||||
|
publishedAt: article.pubDate ? new Date(String(article.pubDate)) : undefined,
|
||||||
|
description: article.description ? String(article.description) : undefined,
|
||||||
|
content: article.content ? String(article.content) : undefined,
|
||||||
|
imageUrl: article.image_url ? String(article.image_url) : undefined,
|
||||||
|
raw: article,
|
||||||
|
}));
|
||||||
|
}
|
||||||
42
src/lib/news/sources/websearch.ts
Normal file
42
src/lib/news/sources/websearch.ts
Normal file
@@ -0,0 +1,42 @@
|
|||||||
|
import { FetchNewsOptions, RawArticle } from '../types';
|
||||||
|
import { getActiveKey } from '@/lib/api-keys';
|
||||||
|
|
||||||
|
export async function fetchWebSearch(options: FetchNewsOptions): Promise<RawArticle[]> {
|
||||||
|
const apiKey = await getActiveKey('WEBSEARCH');
|
||||||
|
if (!apiKey) {
|
||||||
|
console.warn('Web search key not configured');
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
const res = await fetch('https://google.serper.dev/search', {
|
||||||
|
method: 'POST',
|
||||||
|
headers: {
|
||||||
|
'X-API-KEY': apiKey,
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
},
|
||||||
|
body: JSON.stringify({
|
||||||
|
q: options.query,
|
||||||
|
num: Math.min(options.maxResults || 10, 20),
|
||||||
|
tbs: options.from ? `cdr:1,cd_min:${options.from.toISOString().split('T')[0]}` : undefined,
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!res.ok) {
|
||||||
|
const text = await res.text();
|
||||||
|
throw new Error(`Web search error ${res.status}: ${text}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const data = await res.json();
|
||||||
|
|
||||||
|
return (data.news || data.organic || []).map((item: Record<string, unknown>) => ({
|
||||||
|
title: String(item.title || ''),
|
||||||
|
url: String(item.link || item.url || ''),
|
||||||
|
sourceName: String(item.source || 'Web Search'),
|
||||||
|
sourceApiCode: 'websearch',
|
||||||
|
publishedAt: item.date ? new Date(String(item.date)) : undefined,
|
||||||
|
description: item.snippet ? String(item.snippet) : undefined,
|
||||||
|
content: undefined,
|
||||||
|
imageUrl: item.imageUrl ? String(item.imageUrl) : undefined,
|
||||||
|
raw: item,
|
||||||
|
}));
|
||||||
|
}
|
||||||
19
src/lib/news/types.ts
Normal file
19
src/lib/news/types.ts
Normal file
@@ -0,0 +1,19 @@
|
|||||||
|
export type RawArticle = {
|
||||||
|
title: string;
|
||||||
|
url: string;
|
||||||
|
sourceName: string;
|
||||||
|
sourceApiCode: string;
|
||||||
|
publishedAt?: Date;
|
||||||
|
description?: string;
|
||||||
|
content?: string;
|
||||||
|
imageUrl?: string;
|
||||||
|
raw: Record<string, unknown>;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type FetchNewsOptions = {
|
||||||
|
query: string;
|
||||||
|
sources?: string[];
|
||||||
|
maxResults?: number;
|
||||||
|
from?: Date;
|
||||||
|
to?: Date;
|
||||||
|
};
|
||||||
Reference in New Issue
Block a user