add: llms txt pages

apps/web/actions/actions.ts

@@ -0,0 +1,124 @@
+'use server'
+import {
+  calculateProjectScores,
+  getFeaturedProjects,
+  getRecentlyUpdatedProjects
+} from '@/lib/project-utils'
+import { Octokit } from '@octokit/rest'
+import { auth } from '@thedaviddias/auth'
+import { revalidatePath } from 'next/cache'
+
+const owner = 'thedaviddias'
+const repo = 'llms-txt-hub'
+
+export async function submitLlmsTxt(formData: FormData) {
+  const session = await auth()
+
+  if (!session) {
+    throw new Error('Unauthorized')
+  }
+
+  const provider_token = session.provider_token as unknown as { access_token: string }
+  const { access_token } = provider_token
+
+  const octokit = new Octokit({ auth: access_token })
+
+  const name = formData.get('name') as string
+  const description = formData.get('description') as string
+  const website = formData.get('website') as string
+  const llmsUrl = formData.get('llmsUrl') as string
+  const llmsFullUrl = formData.get('llmsFullUrl') as string
+  const githubUsername = session.user.user_metadata.user_name
+
+  const now = new Date().toISOString()
+
+  // Create the content for the new MDX file
+  const content = `---
+name: ${name}
+description: ${description}
+website: ${website}
+llmsUrl: ${llmsUrl}
+llmsFullUrl: ${llmsFullUrl || ''}
+lastUpdated: "${now}"
+score: 0
+---
+
+# ${name}
+
+${description}
+
+## Links
+
+- Website: [${website}](${website})
+- llms.txt: [View llms.txt](${llmsUrl})
+${llmsFullUrl ? `- llms-full.txt: [View llms-full.txt](${llmsFullUrl})` : ''}
+
+## About
+
+Add any additional information about ${name} here.
+`
+
+  try {
+    // Create a new branch
+    const branchName = `submit-${name.toLowerCase().replace(/\s+/g, '-')}-${Date.now()}`
+    const mainRef = await octokit.git.getRef({
+      owner,
+      repo,
+      ref: 'heads/main'
+    })
+
+    await octokit.git.createRef({
+      owner,
+      repo,
+      ref: `refs/heads/${branchName}`,
+      sha: mainRef.data.object.sha
+    })
+
+    // Create the new MDX file in the new branch
+    const filePath = `content/websites/${name.toLowerCase().replace(/\s+/g, '-')}.mdx`
+    await octokit.repos.createOrUpdateFileContents({
+      owner,
+      repo,
+      path: filePath,
+      message: `Add ${name} to llms.txt directory`,
+      content: Buffer.from(content).toString('base64'),
+      branch: branchName
+    })
+
+    // Create a pull request
+    const pr = await octokit.pulls.create({
+      owner,
+      repo,
+      title: `Add ${name} to llms.txt directory`,
+      head: branchName,
+      base: 'main',
+      body: `This PR adds ${name} to the llms.txt directory.
+
+Submitted by: @${githubUsername}
+
+Website: ${website}
+llms.txt: ${llmsUrl}
+${llmsFullUrl ? `llms-full.txt: ${llmsFullUrl}` : ''}
+
+Please review and merge if appropriate.`
+    })
+
+    revalidatePath('/')
+    return { success: true, prUrl: pr.data.html_url }
+  } catch (error) {
+    console.error('Error creating PR:', error)
+    return { success: false, error: 'Failed to create PR' }
+  }
+}
+
+export async function getHomePageData() {
+  const allProjects = await calculateProjectScores()
+  const featuredProjects = getFeaturedProjects(allProjects, 4)
+  const recentlyUpdatedProjects = getRecentlyUpdatedProjects(allProjects, 5)
+
+  return {
+    allProjects,
+    featuredProjects,
+    recentlyUpdatedProjects
+  }
+}

apps/web/app/api/fetch-metadata/route.ts

@@ -0,0 +1,43 @@
+import * as cheerio from 'cheerio'
+import { NextResponse } from 'next/server'
+
+export async function GET(request: Request) {
+  const { searchParams } = new URL(request.url)
+  const domain = searchParams.get('domain')
+
+  if (!domain) {
+    return NextResponse.json({ error: 'Domain is required' }, { status: 400 })
+  }
+
+  try {
+    // Fetch the main page
+    const response = await fetch(domain)
+    const html = await response.text()
+    const $ = cheerio.load(html)
+
+    // Extract metadata
+    const name = $('meta[property="og:site_name"]').attr('content') || $('title').text() || ''
+    const description = $('meta[name="description"]').attr('content') || ''
+
+    // Check for llms.txt
+    const llmsUrl = `${domain}/llms.txt`.replace(/([^:]\/)\/+/g, '$1')
+    const llmsResponse = await fetch(llmsUrl)
+    const llmsExists = llmsResponse.ok
+
+    // Check for llms-full.txt
+    const llmsFullUrl = `${domain}/llms-full.txt`.replace(/([^:]\/)\/+/g, '$1')
+    const llmsFullResponse = await fetch(llmsFullUrl)
+    const llmsFullExists = llmsFullResponse.ok
+
+    return NextResponse.json({
+      name,
+      description,
+      website: domain,
+      llmsUrl: llmsExists ? llmsUrl : '',
+      llmsFullUrl: llmsFullExists ? llmsFullUrl : ''
+    })
+  } catch (error) {
+    console.error('Error fetching metadata:', error)
+    return NextResponse.json({ error: 'Failed to fetch metadata' }, { status: 500 })
+  }
+}

apps/web/app/api/newsletter/route.ts

@@ -0,0 +1,34 @@
+import { env } from '@thedaviddias/config-environment'
+import { logger } from '@thedaviddias/logging'
+import { type NextRequest, NextResponse } from 'next/server'
+
+export async function POST(req: NextRequest) {
+  try {
+    const { email } = await req.json()
+
+    const apiKey = env.MAILERLITE_API_KEY
+
+    if (!apiKey) {
+      return NextResponse.json(
+        { success: false, message: 'Invalid configuration' },
+        { status: 500 }
+      )
+    }
+
+    await fetch('https://connect.mailerlite.com/api/subscribers', {
+      method: 'POST',
+      body: JSON.stringify({ email, groups: ['147152187794916958'] }),
+      headers: {
+        'Content-Type': 'application/json',
+        Accept: 'application/json',
+        Authorization: `Bearer ${apiKey}`
+      }
+    })
+
+    return NextResponse.json({ success: true, message: 'signed up' }, { status: 201 })
+  } catch (error) {
+    logger.error('Error signing up to newsletter')
+
+    return NextResponse.json({ success: false, message: 'internal server error' }, { status: 500 })
+  }
+}

apps/web/app/api/rss-feed/route.ts

@@ -0,0 +1,56 @@
+import { XMLParser } from 'fast-xml-parser'
+import { NextResponse } from 'next/server'
+
+const RSS_FEED_URL = process.env.RSS_FEED_URL || 'https://bg.raindrop.io/rss/public/52790163'
+
+// Segment Config
+export const dynamic = 'force-dynamic'
+export const fetchCache = 'force-no-store'
+export const revalidate = 0
+
+async function fetchWithRetry(url: string, retries = 3): Promise<Response> {
+  for (let i = 0; i < retries; i++) {
+    try {
+      const response = await fetch(url, {
+        next: { revalidate: 3600 },
+        headers: {
+          'User-Agent': 'Mozilla/5.0 (compatible; RSS-Reader/1.0)'
+        }
+      })
+      if (response.ok) return response
+    } catch (error) {
+      if (i === retries - 1) throw error
+      await new Promise(resolve => setTimeout(resolve, 1000 * 2 ** i))
+    }
+  }
+  throw new Error('Failed to fetch after retries')
+}
+
+export async function GET() {
+  try {
+    const response = await fetchWithRetry(RSS_FEED_URL)
+    const xmlData = await response.text()
+    const parser = new XMLParser()
+    const result = parser.parse(xmlData)
+
+    if (!result?.rss?.channel?.item) {
+      return NextResponse.json({ items: [] })
+    }
+
+    const items = Array.isArray(result.rss.channel.item)
+      ? result.rss.channel.item
+      : [result.rss.channel.item]
+
+    const formattedItems = items.map((item: any) => ({
+      title: item.title || '',
+      link: item.link || '',
+      pubDate: item.pubDate || '',
+      description: item.description || ''
+    }))
+
+    return NextResponse.json({ items: formattedItems })
+  } catch (error) {
+    console.error('Failed to fetch or parse RSS feed:', error)
+    return NextResponse.json({ error: 'Failed to fetch or parse RSS feed' }, { status: 500 })
+  }
+}

apps/web/app/api/submit/route.ts

@@ -0,0 +1,101 @@
+import { Octokit } from '@octokit/rest'
+import { auth } from '@thedaviddias/auth'
+import { NextResponse } from 'next/server'
+
+const owner = 'your-github-username'
+const repo = 'your-repo-name'
+
+export async function POST(req: Request) {
+  const session = await auth()
+
+  if (!session) {
+    return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
+  }
+
+  const provider_token = session.provider_token as unknown as { access_token: string }
+  const { access_token } = provider_token
+
+  const octokit = new Octokit({ auth: access_token })
+
+  try {
+    const body = await req.json()
+    const { name, description, website, llmsUrl, llmsFullUrl, githubUsername } = body
+
+    const now = new Date().toISOString()
+
+    // Create the content for the new MDX file
+    const content = `---
+name: ${name}
+description: ${description}
+website: ${website}
+llmsUrl: ${llmsUrl}
+llmsFullUrl: ${llmsFullUrl || ''}
+lastUpdated: "${now}"
+score: 0
+---
+
+# ${name}
+
+${description}
+
+## Links
+
+- Website: [${website}](${website})
+- llms.txt: [View llms.txt](${llmsUrl})
+${llmsFullUrl ? `- llms-full.txt: [View llms-full.txt](${llmsFullUrl})` : ''}
+
+## About
+
+Add any additional information about ${name} here.
+`
+
+    // Create a new branch
+    const branchName = `submit-${name.toLowerCase().replace(/\s+/g, '-')}-${Date.now()}`
+    const mainRef = await octokit.git.getRef({
+      owner,
+      repo,
+      ref: 'heads/main'
+    })
+
+    await octokit.git.createRef({
+      owner,
+      repo,
+      ref: `refs/heads/${branchName}`,
+      sha: mainRef.data.object.sha
+    })
+
+    // Create the new MDX file in the new branch
+    const filePath = `content/websites/${name.toLowerCase().replace(/\s+/g, '-')}.mdx`
+    await octokit.repos.createOrUpdateFileContents({
+      owner,
+      repo,
+      path: filePath,
+      message: `Add ${name} to llms.txt directory`,
+      content: Buffer.from(content).toString('base64'),
+      branch: branchName
+    })
+
+    // Create a pull request
+    const pr = await octokit.pulls.create({
+      owner,
+      repo,
+      title: `Add ${name} to llms.txt directory`,
+      head: branchName,
+      base: 'main',
+      body: `This PR adds ${name} to the llms.txt directory.
+
+Submitted by: @${githubUsername}
+
+Website: ${website}
+llms.txt: ${llmsUrl}
+${llmsFullUrl ? `llms-full.txt: ${llmsFullUrl}` : ''}
+
+Please review and merge if appropriate.`
+    })
+
+    return NextResponse.json({ success: true, prUrl: pr.data.html_url })
+  } catch (error) {
+    console.error('Error creating PR:', error)
+    return NextResponse.json({ success: false, error: 'Failed to create PR' }, { status: 500 })
+  }
+}