feat: add news source adapters and AI article analysis
- Implement adapters for NewsAPI, GNews, NewsData, Serper Web Search, and Brave Search. - Add unified fetchNews aggregator with URL deduplication. - Wire OpenAI-compatible Kimi client for bias classification, ranking, summarization, and topic extraction. - Add AI analysis prompts, response parsing, and fallback logic. - Build ranker combining AI score, source credibility, and recency. - Create admin Discover page and API for manual news discovery.
This commit is contained in:
233
src/app/admin/discover/page.tsx
Normal file
233
src/app/admin/discover/page.tsx
Normal file
@@ -0,0 +1,233 @@
|
||||
'use client';
|
||||
|
||||
import { useEffect, useState } from 'react';
|
||||
import { Bias } from '@prisma/client';
|
||||
|
||||
const availableSources = [
|
||||
{ code: 'newsapi', name: 'NewsAPI' },
|
||||
{ code: 'gnews', name: 'GNews' },
|
||||
{ code: 'websearch', name: 'Web Search (Serper)' },
|
||||
];
|
||||
|
||||
const biasColors: Record<Bias, string> = {
|
||||
LEFT: 'bg-blue-100 text-blue-800 dark:bg-blue-950 dark:text-blue-300',
|
||||
CENTER: 'bg-purple-100 text-purple-800 dark:bg-purple-950 dark:text-purple-300',
|
||||
RIGHT: 'bg-red-100 text-red-800 dark:bg-red-950 dark:text-red-300',
|
||||
};
|
||||
|
||||
type DiscoveredArticle = {
|
||||
success: boolean;
|
||||
article?: {
|
||||
id: string;
|
||||
title: string;
|
||||
summary: string;
|
||||
url: string;
|
||||
sourceName: string;
|
||||
primaryBias: Bias;
|
||||
politicalTags: string[];
|
||||
rankScore: number;
|
||||
credibilityScore: number;
|
||||
topics: string[];
|
||||
status: string;
|
||||
};
|
||||
error?: string;
|
||||
};
|
||||
|
||||
export default function DiscoverPage() {
|
||||
const [query, setQuery] = useState('');
|
||||
const [selectedSources, setSelectedSources] = useState<string[]>(['newsapi', 'gnews']);
|
||||
const [maxResults, setMaxResults] = useState(10);
|
||||
const [autoPublish, setAutoPublish] = useState(false);
|
||||
const [loading, setLoading] = useState(false);
|
||||
const [results, setResults] = useState<DiscoveredArticle[]>([]);
|
||||
const [stats, setStats] = useState<{ found: number; processed: number; failed: number } | null>(null);
|
||||
|
||||
const toggleSource = (code: string) => {
|
||||
setSelectedSources((prev) =>
|
||||
prev.includes(code) ? prev.filter((c) => c !== code) : [...prev, code]
|
||||
);
|
||||
};
|
||||
|
||||
const handleDiscover = async (e: React.FormEvent) => {
|
||||
e.preventDefault();
|
||||
if (selectedSources.length === 0) {
|
||||
alert('Select at least one source');
|
||||
return;
|
||||
}
|
||||
setLoading(true);
|
||||
setResults([]);
|
||||
setStats(null);
|
||||
|
||||
try {
|
||||
const res = await fetch('/api/admin/discover', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
query,
|
||||
sources: selectedSources,
|
||||
maxResults,
|
||||
autoPublish,
|
||||
}),
|
||||
});
|
||||
|
||||
const data = await res.json();
|
||||
if (!res.ok) {
|
||||
alert(data.error || 'Discovery failed');
|
||||
return;
|
||||
}
|
||||
|
||||
setResults(data.results || []);
|
||||
setStats({ found: data.found, processed: data.processed, failed: data.failed });
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div>
|
||||
<h1 className="text-2xl font-bold mb-6">Discover News</h1>
|
||||
|
||||
<div className="bg-white dark:bg-neutral-900 rounded-xl shadow-sm border dark:border-neutral-800 p-6 mb-6">
|
||||
<form onSubmit={handleDiscover} className="space-y-4">
|
||||
<div>
|
||||
<label className="block text-sm font-medium mb-1">Search Query</label>
|
||||
<input
|
||||
type="text"
|
||||
value={query}
|
||||
onChange={(e) => setQuery(e.target.value)}
|
||||
className="w-full px-3 py-2 border rounded-lg dark:bg-neutral-800 dark:border-neutral-700"
|
||||
placeholder="e.g. US politics, climate policy, Supreme Court"
|
||||
required
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<label className="block text-sm font-medium mb-2">Sources</label>
|
||||
<div className="flex flex-wrap gap-3">
|
||||
{availableSources.map((s) => (
|
||||
<label
|
||||
key={s.code}
|
||||
className={`flex items-center gap-2 px-3 py-2 rounded-lg border cursor-pointer ${
|
||||
selectedSources.includes(s.code)
|
||||
? 'bg-blue-50 border-blue-300 dark:bg-blue-950 dark:border-blue-700'
|
||||
: 'bg-white dark:bg-neutral-900 border-neutral-200 dark:border-neutral-700'
|
||||
}`}
|
||||
>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={selectedSources.includes(s.code)}
|
||||
onChange={() => toggleSource(s.code)}
|
||||
/>
|
||||
<span className="text-sm">{s.name}</span>
|
||||
</label>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="grid grid-cols-1 md:grid-cols-3 gap-4 items-end">
|
||||
<div>
|
||||
<label className="block text-sm font-medium mb-1">Max Results per Source</label>
|
||||
<input
|
||||
type="number"
|
||||
min={1}
|
||||
max={50}
|
||||
value={maxResults}
|
||||
onChange={(e) => setMaxResults(Number(e.target.value))}
|
||||
className="w-full px-3 py-2 border rounded-lg dark:bg-neutral-800 dark:border-neutral-700"
|
||||
/>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<input
|
||||
id="autoPublish"
|
||||
type="checkbox"
|
||||
checked={autoPublish}
|
||||
onChange={(e) => setAutoPublish(e.target.checked)}
|
||||
/>
|
||||
<label htmlFor="autoPublish" className="text-sm font-medium">
|
||||
Auto-publish discovered articles
|
||||
</label>
|
||||
</div>
|
||||
<div className="text-right">
|
||||
<button
|
||||
type="submit"
|
||||
disabled={loading}
|
||||
className="px-4 py-2 bg-blue-600 hover:bg-blue-700 text-white rounded-lg font-medium disabled:opacity-50"
|
||||
>
|
||||
{loading ? 'Discovering...' : 'Discover News'}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
|
||||
{stats && (
|
||||
<div className="mb-4 text-sm text-neutral-600 dark:text-neutral-400">
|
||||
Found {stats.found} raw articles. Processed {stats.processed}, failed {stats.failed}.
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="space-y-4">
|
||||
{results.map((result, idx) => {
|
||||
if (!result.success || !result.article) {
|
||||
return (
|
||||
<div
|
||||
key={idx}
|
||||
className="bg-red-50 dark:bg-red-950 border border-red-200 dark:border-red-900 rounded-xl p-4 text-sm text-red-700 dark:text-red-300"
|
||||
>
|
||||
Failed: {result.error}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
const a = result.article;
|
||||
return (
|
||||
<div
|
||||
key={a.id}
|
||||
className="bg-white dark:bg-neutral-900 rounded-xl shadow-sm border dark:border-neutral-800 p-5"
|
||||
>
|
||||
<div className="flex flex-wrap items-start justify-between gap-3 mb-2">
|
||||
<h3 className="text-lg font-semibold">
|
||||
<a href={a.url} target="_blank" rel="noopener noreferrer" className="hover:underline">
|
||||
{a.title}
|
||||
</a>
|
||||
</h3>
|
||||
<span
|
||||
className={`px-2 py-1 rounded-full text-xs font-medium ${biasColors[a.primaryBias]}`}
|
||||
>
|
||||
{a.primaryBias}
|
||||
</span>
|
||||
</div>
|
||||
<p className="text-sm text-neutral-600 dark:text-neutral-400 mb-3 line-clamp-3">
|
||||
{a.summary}
|
||||
</p>
|
||||
<div className="flex flex-wrap gap-2 mb-3">
|
||||
{a.politicalTags.map((tag) => (
|
||||
<span
|
||||
key={tag}
|
||||
className="px-2 py-1 rounded-md text-xs bg-neutral-100 dark:bg-neutral-800"
|
||||
>
|
||||
{tag}
|
||||
</span>
|
||||
))}
|
||||
{a.topics.slice(0, 5).map((topic) => (
|
||||
<span
|
||||
key={topic}
|
||||
className="px-2 py-1 rounded-md text-xs bg-blue-50 dark:bg-blue-950 text-blue-700 dark:text-blue-300"
|
||||
>
|
||||
{topic}
|
||||
</span>
|
||||
))}
|
||||
</div>
|
||||
<div className="flex flex-wrap gap-4 text-xs text-neutral-500 dark:text-neutral-400">
|
||||
<span>Source: {a.sourceName}</span>
|
||||
<span>Rank: {a.rankScore}/10</span>
|
||||
<span>Credibility: {a.credibilityScore}/10</span>
|
||||
<span>Status: {a.status}</span>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
58
src/app/api/admin/discover/route.ts
Normal file
58
src/app/api/admin/discover/route.ts
Normal file
@@ -0,0 +1,58 @@
|
||||
import { NextResponse } from 'next/server';
|
||||
import { fetchNews } from '@/lib/news';
|
||||
import { processArticle } from '@/lib/ai/processor';
|
||||
import { saveProcessedArticle } from '@/lib/articles';
|
||||
|
||||
export async function POST(request: Request) {
|
||||
try {
|
||||
const body = await request.json();
|
||||
const { query, sources, maxResults, autoPublish } = body;
|
||||
|
||||
if (!query || typeof query !== 'string') {
|
||||
return NextResponse.json({ error: 'Query is required' }, { status: 400 });
|
||||
}
|
||||
|
||||
const rawArticles = await fetchNews({
|
||||
query,
|
||||
sources: sources || undefined,
|
||||
maxResults: maxResults || 20,
|
||||
});
|
||||
|
||||
const processed = await Promise.allSettled(
|
||||
rawArticles.map(async (raw) => {
|
||||
const analyzed = await processArticle(raw);
|
||||
const status = autoPublish ? 'PUBLISHED' : 'DRAFT';
|
||||
const saved = await saveProcessedArticle(analyzed, status);
|
||||
return {
|
||||
...analyzed,
|
||||
id: saved.id,
|
||||
status,
|
||||
politicalTags: analyzed.politicalTags,
|
||||
topics: analyzed.topics,
|
||||
};
|
||||
})
|
||||
);
|
||||
|
||||
const results = processed.map((result) =>
|
||||
result.status === 'fulfilled'
|
||||
? { success: true, article: result.value }
|
||||
: { success: false, error: String(result.reason) }
|
||||
);
|
||||
|
||||
const succeeded = results.filter((r) => r.success);
|
||||
const failed = results.filter((r) => !r.success);
|
||||
|
||||
return NextResponse.json({
|
||||
found: rawArticles.length,
|
||||
processed: succeeded.length,
|
||||
failed: failed.length,
|
||||
results,
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Discover error:', error);
|
||||
return NextResponse.json(
|
||||
{ error: String(error) },
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
17
src/lib/ai/client.ts
Normal file
17
src/lib/ai/client.ts
Normal file
@@ -0,0 +1,17 @@
|
||||
import OpenAI from 'openai';
|
||||
import { getActiveKey } from '../api-keys';
|
||||
|
||||
export async function getAiClient(): Promise<OpenAI> {
|
||||
const apiKey = await getActiveKey('KIMI');
|
||||
const baseURL = process.env.KIMI_API_URL || 'https://api.moonshot.ai/v1';
|
||||
|
||||
if (!apiKey) {
|
||||
throw new Error('Kimi API key not configured. Add one in Admin > API Keys.');
|
||||
}
|
||||
|
||||
return new OpenAI({ apiKey, baseURL, timeout: 60_000, maxRetries: 1 });
|
||||
}
|
||||
|
||||
export function getAiModel(): string {
|
||||
return process.env.KIMI_MODEL || 'kimi-k2.5';
|
||||
}
|
||||
95
src/lib/ai/processor.ts
Normal file
95
src/lib/ai/processor.ts
Normal file
@@ -0,0 +1,95 @@
|
||||
import { RawArticle } from '../news/types';
|
||||
import { getAiClient, getAiModel } from './client';
|
||||
import { buildArticleAnalysisPrompt } from './prompts';
|
||||
import { Bias } from '@prisma/client';
|
||||
|
||||
export type ProcessedArticle = {
|
||||
title: string;
|
||||
summary: string;
|
||||
url: string;
|
||||
sourceName: string;
|
||||
sourceApiCode: string;
|
||||
publishedAt?: Date;
|
||||
imageUrl?: string;
|
||||
primaryBias: Bias;
|
||||
biasConfidence: number;
|
||||
politicalTags: string[];
|
||||
rankScore: number;
|
||||
credibilityScore: number;
|
||||
topics: string[];
|
||||
aiExplanation: string;
|
||||
raw: Record<string, unknown>;
|
||||
};
|
||||
|
||||
function safeParseBias(value: string): Bias {
|
||||
const upper = String(value).toUpperCase();
|
||||
if (upper === 'LEFT' || upper === 'CENTER' || upper === 'RIGHT') {
|
||||
return upper as Bias;
|
||||
}
|
||||
return 'CENTER';
|
||||
}
|
||||
|
||||
function safeParseNumber(value: unknown, fallback: number, min = 0, max = 10): number {
|
||||
const num = Number(value);
|
||||
if (Number.isNaN(num)) return fallback;
|
||||
return Math.max(min, Math.min(max, num));
|
||||
}
|
||||
|
||||
export async function processArticle(
|
||||
raw: RawArticle,
|
||||
options?: { model?: string }
|
||||
): Promise<ProcessedArticle> {
|
||||
const client = await getAiClient();
|
||||
const prompt = buildArticleAnalysisPrompt({
|
||||
title: raw.title,
|
||||
sourceName: raw.sourceName,
|
||||
description: raw.description,
|
||||
content: raw.content,
|
||||
url: raw.url,
|
||||
});
|
||||
|
||||
const res = await client.chat.completions.create({
|
||||
model: options?.model || getAiModel(),
|
||||
messages: [{ role: 'user', content: prompt }],
|
||||
});
|
||||
|
||||
const content = res.choices[0]?.message?.content?.trim() || '{}';
|
||||
let parsed: Record<string, unknown> = {};
|
||||
|
||||
try {
|
||||
const cleaned = content.replace(/^```json\s*|\s*```$/g, '');
|
||||
parsed = JSON.parse(cleaned);
|
||||
} catch (err) {
|
||||
console.error('Failed to parse AI response:', content, err);
|
||||
parsed = {};
|
||||
}
|
||||
|
||||
const politicalTags = Array.isArray(parsed.politicalTags)
|
||||
? parsed.politicalTags.map(String).filter((t) => ['liberal', 'moderate', 'conservative'].includes(t.toLowerCase()))
|
||||
: [];
|
||||
|
||||
if (politicalTags.length === 0) {
|
||||
const bias = safeParseBias(String(parsed.primaryBias));
|
||||
if (bias === 'LEFT') politicalTags.push('liberal');
|
||||
if (bias === 'CENTER') politicalTags.push('moderate');
|
||||
if (bias === 'RIGHT') politicalTags.push('conservative');
|
||||
}
|
||||
|
||||
return {
|
||||
title: raw.title,
|
||||
summary: String(parsed.summary || raw.description || ''),
|
||||
url: raw.url,
|
||||
sourceName: raw.sourceName,
|
||||
sourceApiCode: raw.sourceApiCode,
|
||||
publishedAt: raw.publishedAt,
|
||||
imageUrl: raw.imageUrl,
|
||||
primaryBias: safeParseBias(parsed.primaryBias as string),
|
||||
biasConfidence: safeParseNumber(parsed.biasConfidence, 0.5, 0, 1),
|
||||
politicalTags,
|
||||
rankScore: safeParseNumber(parsed.rankScore, 5, 1, 10),
|
||||
credibilityScore: safeParseNumber(parsed.credibilityScore, 5, 1, 10),
|
||||
topics: Array.isArray(parsed.topics) ? parsed.topics.map(String) : [],
|
||||
aiExplanation: String(parsed.explanation || ''),
|
||||
raw: raw.raw,
|
||||
};
|
||||
}
|
||||
55
src/lib/ai/prompts.ts
Normal file
55
src/lib/ai/prompts.ts
Normal file
@@ -0,0 +1,55 @@
|
||||
export function buildArticleAnalysisPrompt(article: {
|
||||
title: string;
|
||||
sourceName: string;
|
||||
description?: string;
|
||||
content?: string;
|
||||
url: string;
|
||||
}): string {
|
||||
const text = [article.title, article.description, article.content]
|
||||
.filter(Boolean)
|
||||
.join('\n\n');
|
||||
|
||||
return `Analyze the following news article and return ONLY a JSON object with no markdown formatting.
|
||||
|
||||
Article URL: ${article.url}
|
||||
Source: ${article.sourceName}
|
||||
Text:
|
||||
"""
|
||||
${text.slice(0, 12000)}
|
||||
"""
|
||||
|
||||
Return JSON in this exact shape:
|
||||
{
|
||||
"summary": "A concise 2-3 sentence summary of the article.",
|
||||
"primaryBias": "LEFT" | "CENTER" | "RIGHT",
|
||||
"biasConfidence": 0.0 to 1.0,
|
||||
"politicalTags": ["liberal" | "moderate" | "conservative"],
|
||||
"rankScore": 1 to 10,
|
||||
"credibilityScore": 1 to 10,
|
||||
"topics": ["topic1", "topic2"],
|
||||
"explanation": "One sentence explaining the bias classification."
|
||||
}
|
||||
|
||||
Definitions:
|
||||
- LEFT: favors progressive/liberal policy, Democrats, or left-wing framing.
|
||||
- CENTER: neutral, balanced, wire-service, or mainstream non-partisan framing.
|
||||
- RIGHT: favors conservative/traditional policy, Republicans, or right-wing framing.
|
||||
politicalTags should include one or more of: liberal, moderate, conservative.
|
||||
rankScore reflects how important/significant the story is (1 = trivial, 10 = major).
|
||||
credibilityScore reflects source reliability (1 = unreliable, 10 = highly credible).`;
|
||||
}
|
||||
|
||||
export function buildNewsSearchPrompt(query: string): string {
|
||||
return `Find recent news articles about "${query}". Return ONLY a JSON array of objects with no markdown formatting.
|
||||
|
||||
Each object must have:
|
||||
{
|
||||
"title": "Article title",
|
||||
"url": "Article URL",
|
||||
"sourceName": "Source name",
|
||||
"publishedAt": "ISO date string or null",
|
||||
"description": "Brief description or snippet"
|
||||
}
|
||||
|
||||
Return at most 10 articles. If you cannot browse the live web, return an empty array [].`;
|
||||
}
|
||||
21
src/lib/ai/ranker.ts
Normal file
21
src/lib/ai/ranker.ts
Normal file
@@ -0,0 +1,21 @@
|
||||
export function computeFinalRankScore({
|
||||
aiRankScore,
|
||||
credibilityScore,
|
||||
publishedAt,
|
||||
}: {
|
||||
aiRankScore: number;
|
||||
credibilityScore: number;
|
||||
publishedAt?: Date;
|
||||
}): number {
|
||||
// Base score: 60% AI rank, 40% credibility
|
||||
let score = aiRankScore * 0.6 + credibilityScore * 0.4;
|
||||
|
||||
// Recency boost: up to +1.5 for articles published in last 24h, decaying over 7 days
|
||||
if (publishedAt) {
|
||||
const ageHours = (Date.now() - publishedAt.getTime()) / (1000 * 60 * 60);
|
||||
const recencyBoost = Math.max(0, 1.5 * (1 - ageHours / (24 * 7)));
|
||||
score += recencyBoost;
|
||||
}
|
||||
|
||||
return Math.min(10, Math.max(1, Number(score.toFixed(2))));
|
||||
}
|
||||
106
src/lib/articles.ts
Normal file
106
src/lib/articles.ts
Normal file
@@ -0,0 +1,106 @@
|
||||
import { prisma } from './db';
|
||||
import { ProcessedArticle } from './ai/processor';
|
||||
import { computeFinalRankScore } from './ai/ranker';
|
||||
import { ArticleStatus, Bias, Prisma } from '@prisma/client';
|
||||
|
||||
export async function findOrCreateSource(apiCode: string, name: string) {
|
||||
return prisma.source.upsert({
|
||||
where: { apiCode },
|
||||
update: {},
|
||||
create: { apiCode, name },
|
||||
});
|
||||
}
|
||||
|
||||
export async function saveProcessedArticle(
|
||||
processed: ProcessedArticle,
|
||||
status: ArticleStatus = 'DRAFT'
|
||||
) {
|
||||
const source = await findOrCreateSource(processed.sourceApiCode, processed.sourceName);
|
||||
|
||||
const existing = await prisma.article.findUnique({
|
||||
where: { url: processed.url },
|
||||
});
|
||||
|
||||
const finalRank = computeFinalRankScore({
|
||||
aiRankScore: processed.rankScore,
|
||||
credibilityScore: processed.credibilityScore,
|
||||
publishedAt: processed.publishedAt,
|
||||
});
|
||||
|
||||
const data: Prisma.ArticleUncheckedCreateInput = {
|
||||
title: processed.title,
|
||||
summary: processed.summary,
|
||||
url: processed.url,
|
||||
sourceId: source.id,
|
||||
imageUrl: processed.imageUrl,
|
||||
publishedAt: processed.publishedAt,
|
||||
primaryBias: processed.primaryBias as Bias,
|
||||
biasConfidence: processed.biasConfidence,
|
||||
politicalTags: JSON.stringify(processed.politicalTags),
|
||||
rankScore: finalRank,
|
||||
credibilityScore: processed.credibilityScore,
|
||||
status,
|
||||
topics: JSON.stringify(processed.topics),
|
||||
rawData: JSON.stringify(processed.raw),
|
||||
aiExplanation: processed.aiExplanation,
|
||||
};
|
||||
|
||||
if (existing) {
|
||||
return prisma.article.update({
|
||||
where: { id: existing.id },
|
||||
data: {
|
||||
...data,
|
||||
id: undefined,
|
||||
url: undefined,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return prisma.article.create({ data });
|
||||
}
|
||||
|
||||
export async function getPublishedArticles(filters?: {
|
||||
bias?: Bias;
|
||||
query?: string;
|
||||
sourceId?: string;
|
||||
topic?: string;
|
||||
from?: Date;
|
||||
to?: Date;
|
||||
limit?: number;
|
||||
offset?: number;
|
||||
}) {
|
||||
const where: Prisma.ArticleWhereInput = { status: 'PUBLISHED' };
|
||||
|
||||
if (filters?.bias) where.primaryBias = filters.bias;
|
||||
if (filters?.sourceId) where.sourceId = filters.sourceId;
|
||||
if (filters?.topic) {
|
||||
where.topics = { contains: `"${filters.topic}"` };
|
||||
}
|
||||
if (filters?.from || filters?.to) {
|
||||
where.publishedAt = {};
|
||||
if (filters.from) where.publishedAt.gte = filters.from;
|
||||
if (filters.to) where.publishedAt.lte = filters.to;
|
||||
}
|
||||
if (filters?.query) {
|
||||
where.OR = [
|
||||
{ title: { contains: filters.query } },
|
||||
{ summary: { contains: filters.query } },
|
||||
{ topics: { contains: filters.query } },
|
||||
];
|
||||
}
|
||||
|
||||
return prisma.article.findMany({
|
||||
where,
|
||||
orderBy: [{ rankScore: 'desc' }, { publishedAt: 'desc' }],
|
||||
take: filters?.limit ?? 50,
|
||||
skip: filters?.offset ?? 0,
|
||||
include: { source: true },
|
||||
});
|
||||
}
|
||||
|
||||
export async function getArticleById(id: string) {
|
||||
return prisma.article.findUnique({
|
||||
where: { id },
|
||||
include: { source: true },
|
||||
});
|
||||
}
|
||||
46
src/lib/news/index.ts
Normal file
46
src/lib/news/index.ts
Normal file
@@ -0,0 +1,46 @@
|
||||
import { FetchNewsOptions, RawArticle } from './types';
|
||||
import { fetchNewsApi } from './sources/newsapi';
|
||||
import { fetchGNews } from './sources/gnews';
|
||||
import { fetchWebSearch } from './sources/websearch';
|
||||
import { fetchNewsData } from './sources/newsdata';
|
||||
import { fetchBrave } from './sources/brave';
|
||||
|
||||
const sourceMap: Record<string, (opts: FetchNewsOptions) => Promise<RawArticle[]>> = {
|
||||
newsapi: fetchNewsApi,
|
||||
gnews: fetchGNews,
|
||||
websearch: fetchWebSearch,
|
||||
newsdata: fetchNewsData,
|
||||
brave: fetchBrave,
|
||||
};
|
||||
|
||||
export async function fetchNews(options: FetchNewsOptions): Promise<RawArticle[]> {
|
||||
const sources = options.sources && options.sources.length > 0
|
||||
? options.sources
|
||||
: Object.keys(sourceMap);
|
||||
|
||||
const results = await Promise.allSettled(
|
||||
sources.map(async (source) => {
|
||||
const fetcher = sourceMap[source.toLowerCase()];
|
||||
if (!fetcher) throw new Error(`Unknown news source: ${source}`);
|
||||
return fetcher(options);
|
||||
})
|
||||
);
|
||||
|
||||
const articles: RawArticle[] = [];
|
||||
results.forEach((result, index) => {
|
||||
if (result.status === 'fulfilled') {
|
||||
articles.push(...result.value);
|
||||
} else {
|
||||
console.error(`Source ${sources[index]} failed:`, result.reason);
|
||||
}
|
||||
});
|
||||
|
||||
// Deduplicate by URL
|
||||
const seen = new Set<string>();
|
||||
return articles.filter((article) => {
|
||||
const normalized = article.url.split('?')[0].toLowerCase();
|
||||
if (seen.has(normalized)) return false;
|
||||
seen.add(normalized);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
80
src/lib/news/sources/brave.ts
Normal file
80
src/lib/news/sources/brave.ts
Normal file
@@ -0,0 +1,80 @@
|
||||
import { FetchNewsOptions, RawArticle } from '../types';
|
||||
import { getActiveKey } from '@/lib/api-keys';
|
||||
|
||||
function parseBraveAge(age?: string): Date | undefined {
|
||||
if (!age) return undefined;
|
||||
// Brave returns relative strings like "1 hour ago", "2 days ago", "Jun 16, 2026"
|
||||
// We can only reliably parse the ISO-like or simple relative forms here.
|
||||
// For now, fall back to 'now' if parsing fails.
|
||||
try {
|
||||
const match = age.match(/(\d+)\s+(minute|hour|day|week|month|year)s?\s+ago/i);
|
||||
if (match) {
|
||||
const value = parseInt(match[1], 10);
|
||||
const unit = match[2].toLowerCase();
|
||||
const now = new Date();
|
||||
switch (unit) {
|
||||
case 'minute': now.setMinutes(now.getMinutes() - value); break;
|
||||
case 'hour': now.setHours(now.getHours() - value); break;
|
||||
case 'day': now.setDate(now.getDate() - value); break;
|
||||
case 'week': now.setDate(now.getDate() - value * 7); break;
|
||||
case 'month': now.setMonth(now.getMonth() - value); break;
|
||||
case 'year': now.setFullYear(now.getFullYear() - value); break;
|
||||
}
|
||||
return now;
|
||||
}
|
||||
const parsed = new Date(age);
|
||||
if (!isNaN(parsed.getTime())) return parsed;
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export async function fetchBrave(options: FetchNewsOptions): Promise<RawArticle[]> {
|
||||
const apiKey = await getActiveKey('BRAVE');
|
||||
if (!apiKey) {
|
||||
console.warn('Brave Search key not configured');
|
||||
return [];
|
||||
}
|
||||
|
||||
const params = new URLSearchParams({
|
||||
q: options.query,
|
||||
count: String(Math.min(options.maxResults || 10, 20)),
|
||||
search_lang: 'en',
|
||||
text_decorations: 'false',
|
||||
});
|
||||
|
||||
if (options.from) {
|
||||
params.set('freshness', `dtf:${options.from.toISOString().split('T')[0]}_9999-12-31`);
|
||||
}
|
||||
|
||||
const res = await fetch(`https://api.search.brave.com/res/v1/news/search?${params.toString()}`, {
|
||||
headers: {
|
||||
'X-Subscription-Token': apiKey,
|
||||
'Accept': 'application/json',
|
||||
},
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
const text = await res.text();
|
||||
throw new Error(`Brave search error ${res.status}: ${text}`);
|
||||
}
|
||||
|
||||
const data = await res.json();
|
||||
|
||||
return (data.results || []).map((item: Record<string, unknown>) => {
|
||||
const meta = item.meta as Record<string, unknown> | undefined;
|
||||
const metaUrl = meta?.url as Record<string, unknown> | undefined;
|
||||
return {
|
||||
title: String(item.title || ''),
|
||||
url: String(item.url || ''),
|
||||
sourceName: String(metaUrl?.hostname || item.source || 'Brave Search'),
|
||||
sourceApiCode: 'brave',
|
||||
publishedAt: parseBraveAge(item.age ? String(item.age) : undefined),
|
||||
description: item.description ? String(item.description) : undefined,
|
||||
content: undefined,
|
||||
imageUrl: metaUrl?.favicon ? String(metaUrl.favicon) : undefined,
|
||||
raw: item,
|
||||
};
|
||||
});
|
||||
}
|
||||
40
src/lib/news/sources/gnews.ts
Normal file
40
src/lib/news/sources/gnews.ts
Normal file
@@ -0,0 +1,40 @@
|
||||
import { FetchNewsOptions, RawArticle } from '../types';
|
||||
import { getActiveKey } from '@/lib/api-keys';
|
||||
|
||||
export async function fetchGNews(options: FetchNewsOptions): Promise<RawArticle[]> {
|
||||
const apiKey = await getActiveKey('GNEWS');
|
||||
if (!apiKey) {
|
||||
console.warn('GNews key not configured');
|
||||
return [];
|
||||
}
|
||||
|
||||
const url = new URL('https://gnews.io/api/v4/search');
|
||||
url.searchParams.set('q', options.query);
|
||||
url.searchParams.set('apikey', apiKey);
|
||||
url.searchParams.set('max', String(Math.min(options.maxResults || 20, 100)));
|
||||
url.searchParams.set('lang', 'en');
|
||||
url.searchParams.set('sortby', 'publishedAt');
|
||||
|
||||
if (options.from) url.searchParams.set('from', options.from.toISOString());
|
||||
if (options.to) url.searchParams.set('to', options.to.toISOString());
|
||||
|
||||
const res = await fetch(url.toString());
|
||||
if (!res.ok) {
|
||||
const text = await res.text();
|
||||
throw new Error(`GNews error ${res.status}: ${text}`);
|
||||
}
|
||||
|
||||
const data = await res.json();
|
||||
|
||||
return (data.articles || []).map((article: Record<string, unknown>) => ({
|
||||
title: String(article.title || ''),
|
||||
url: String(article.url || ''),
|
||||
sourceName: String((article.source as Record<string, unknown>)?.name || 'GNews'),
|
||||
sourceApiCode: 'gnews',
|
||||
publishedAt: article.publishedAt ? new Date(String(article.publishedAt)) : undefined,
|
||||
description: article.description ? String(article.description) : undefined,
|
||||
content: article.content ? String(article.content) : undefined,
|
||||
imageUrl: article.image ? String(article.image) : undefined,
|
||||
raw: article,
|
||||
}));
|
||||
}
|
||||
43
src/lib/news/sources/newsapi.ts
Normal file
43
src/lib/news/sources/newsapi.ts
Normal file
@@ -0,0 +1,43 @@
|
||||
import { FetchNewsOptions, RawArticle } from '../types';
|
||||
import { getActiveKey } from '@/lib/api-keys';
|
||||
|
||||
export async function fetchNewsApi(options: FetchNewsOptions): Promise<RawArticle[]> {
|
||||
const apiKey = await getActiveKey('NEWSAPI');
|
||||
if (!apiKey) {
|
||||
console.warn('NewsAPI key not configured');
|
||||
return [];
|
||||
}
|
||||
|
||||
const url = new URL('https://newsapi.org/v2/everything');
|
||||
url.searchParams.set('q', options.query);
|
||||
url.searchParams.set('apiKey', apiKey);
|
||||
url.searchParams.set('pageSize', String(Math.min(options.maxResults || 20, 100)));
|
||||
url.searchParams.set('sortBy', 'publishedAt');
|
||||
url.searchParams.set('language', 'en');
|
||||
|
||||
if (options.from) url.searchParams.set('from', options.from.toISOString());
|
||||
if (options.to) url.searchParams.set('to', options.to.toISOString());
|
||||
|
||||
const res = await fetch(url.toString());
|
||||
if (!res.ok) {
|
||||
const text = await res.text();
|
||||
throw new Error(`NewsAPI error ${res.status}: ${text}`);
|
||||
}
|
||||
|
||||
const data = await res.json();
|
||||
if (data.status !== 'ok') {
|
||||
throw new Error(`NewsAPI error: ${data.message || 'unknown'}`);
|
||||
}
|
||||
|
||||
return (data.articles || []).map((article: Record<string, unknown>) => ({
|
||||
title: String(article.title || ''),
|
||||
url: String(article.url || ''),
|
||||
sourceName: String((article.source as Record<string, unknown>)?.name || 'NewsAPI'),
|
||||
sourceApiCode: 'newsapi',
|
||||
publishedAt: article.publishedAt ? new Date(String(article.publishedAt)) : undefined,
|
||||
description: article.description ? String(article.description) : undefined,
|
||||
content: article.content ? String(article.content) : undefined,
|
||||
imageUrl: article.urlToImage ? String(article.urlToImage) : undefined,
|
||||
raw: article,
|
||||
}));
|
||||
}
|
||||
42
src/lib/news/sources/newsdata.ts
Normal file
42
src/lib/news/sources/newsdata.ts
Normal file
@@ -0,0 +1,42 @@
|
||||
import { FetchNewsOptions, RawArticle } from '../types';
|
||||
import { getActiveKey } from '@/lib/api-keys';
|
||||
|
||||
export async function fetchNewsData(options: FetchNewsOptions): Promise<RawArticle[]> {
|
||||
const apiKey = await getActiveKey('NEWSDATA');
|
||||
if (!apiKey) {
|
||||
console.warn('NewsData key not configured');
|
||||
return [];
|
||||
}
|
||||
|
||||
const url = new URL('https://newsdata.io/api/1/latest');
|
||||
url.searchParams.set('q', options.query);
|
||||
url.searchParams.set('apikey', apiKey);
|
||||
url.searchParams.set('size', String(Math.min(options.maxResults || 10, 50)));
|
||||
url.searchParams.set('language', 'en');
|
||||
|
||||
if (options.from) url.searchParams.set('from_date', options.from.toISOString().split('T')[0]);
|
||||
if (options.to) url.searchParams.set('to_date', options.to.toISOString().split('T')[0]);
|
||||
|
||||
const res = await fetch(url.toString());
|
||||
if (!res.ok) {
|
||||
const text = await res.text();
|
||||
throw new Error(`NewsData error ${res.status}: ${text}`);
|
||||
}
|
||||
|
||||
const data = await res.json();
|
||||
if (data.status !== 'success') {
|
||||
throw new Error(`NewsData error: ${data.message || data.status || 'unknown'}`);
|
||||
}
|
||||
|
||||
return (data.results || []).map((article: Record<string, unknown>) => ({
|
||||
title: String(article.title || ''),
|
||||
url: String(article.link || ''),
|
||||
sourceName: String(article.source_id || 'NewsData'),
|
||||
sourceApiCode: 'newsdata',
|
||||
publishedAt: article.pubDate ? new Date(String(article.pubDate)) : undefined,
|
||||
description: article.description ? String(article.description) : undefined,
|
||||
content: article.content ? String(article.content) : undefined,
|
||||
imageUrl: article.image_url ? String(article.image_url) : undefined,
|
||||
raw: article,
|
||||
}));
|
||||
}
|
||||
42
src/lib/news/sources/websearch.ts
Normal file
42
src/lib/news/sources/websearch.ts
Normal file
@@ -0,0 +1,42 @@
|
||||
import { FetchNewsOptions, RawArticle } from '../types';
|
||||
import { getActiveKey } from '@/lib/api-keys';
|
||||
|
||||
export async function fetchWebSearch(options: FetchNewsOptions): Promise<RawArticle[]> {
|
||||
const apiKey = await getActiveKey('WEBSEARCH');
|
||||
if (!apiKey) {
|
||||
console.warn('Web search key not configured');
|
||||
return [];
|
||||
}
|
||||
|
||||
const res = await fetch('https://google.serper.dev/search', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'X-API-KEY': apiKey,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
q: options.query,
|
||||
num: Math.min(options.maxResults || 10, 20),
|
||||
tbs: options.from ? `cdr:1,cd_min:${options.from.toISOString().split('T')[0]}` : undefined,
|
||||
}),
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
const text = await res.text();
|
||||
throw new Error(`Web search error ${res.status}: ${text}`);
|
||||
}
|
||||
|
||||
const data = await res.json();
|
||||
|
||||
return (data.news || data.organic || []).map((item: Record<string, unknown>) => ({
|
||||
title: String(item.title || ''),
|
||||
url: String(item.link || item.url || ''),
|
||||
sourceName: String(item.source || 'Web Search'),
|
||||
sourceApiCode: 'websearch',
|
||||
publishedAt: item.date ? new Date(String(item.date)) : undefined,
|
||||
description: item.snippet ? String(item.snippet) : undefined,
|
||||
content: undefined,
|
||||
imageUrl: item.imageUrl ? String(item.imageUrl) : undefined,
|
||||
raw: item,
|
||||
}));
|
||||
}
|
||||
19
src/lib/news/types.ts
Normal file
19
src/lib/news/types.ts
Normal file
@@ -0,0 +1,19 @@
|
||||
export type RawArticle = {
|
||||
title: string;
|
||||
url: string;
|
||||
sourceName: string;
|
||||
sourceApiCode: string;
|
||||
publishedAt?: Date;
|
||||
description?: string;
|
||||
content?: string;
|
||||
imageUrl?: string;
|
||||
raw: Record<string, unknown>;
|
||||
};
|
||||
|
||||
export type FetchNewsOptions = {
|
||||
query: string;
|
||||
sources?: string[];
|
||||
maxResults?: number;
|
||||
from?: Date;
|
||||
to?: Date;
|
||||
};
|
||||
Reference in New Issue
Block a user