Folo Flavored Feed Spec

Feed definition in Folo

The following are the data types of feeds and entries in Folo.

type FeedModel = {
    url: string;
    image: string | null;
    description: string | null;
    title: string | null;
    siteUrl: string | null;
    lastModifiedHeader: string | null;
    etagHeader: string | null;
    ttl: number | null;

    // internal
    id: string;
    checkedAt: Date;
    errorMessage: string | null;
    errorAt: Date | null;
    ownerUserId: string | null;
    language: string | null;
    migrateTo: string | null;
}

type EntryModel = {
    guid: string;
    publishedAt: Date;
    description: string | null;
    title: string | null;
    content: string | null;
    author: string | null;
    url: string | null;
    categories: string[] | null;
    authorUrl: string | null;
    authorAvatar: string | null;
    attachments: {
        url: string
        duration_in_seconds?: number
        mime_type?: string
        size_in_bytes?: number
        title?: string
    }[] | null;
    extra: {
        links?: {
            url: string
            type: string
            content_html?: string
        }[]
    } | null;
    media: {
        url: string
        type: "photo" | "video"
        preview_image_url?: string

        // internal
        width?: number
        height?: number
        blurhash?: string
    }[] | null;

    // internal
    id: string;
    insertedAt: Date;
    feedId: string;
    language: string | null;
}

Conversion logics

Folo supports the parsing of JSON Feed, RSS, Email, Wehbook. The complete parsing logic is detailed below.

Convert from JSON Feed

import { getPublishedAt, safeTTL } from "@/lib/helpers"

import { normalizeDurationInSeconds } from "./normalize-attachments"
import type { FeedModelizerFunction } from "./types"

/**
 * Convert JSON Feed to DB model
 */
export const modelizeJsonFeed: FeedModelizerFunction = async (data: JsonFeedV1.Feed | JsonFeedV1_1.Feed) => {
  const now = Date.now()

  const feed = {
    title: data.title,
    description: data.description,
    siteUrl: data.home_page_url,
    image: data.icon,
    checkedAt: new Date(now),
    ttl: safeTTL(),

    follow_challenge: data.follow_challenge,
  }

  const entries = (data.items || [])
    .map((item, index) => {
      if ("author" in item) {
        (item as any).authors = [item.author]
      }
      return {
        title: item.title,
        url: item.url,
        guid: `${item.id}`,
        content: item.content_html || item.summary,
        author: "authors" in item ? item.authors?.[0].name : undefined,
        authorUrl: "authors" in item ? item.authors?.[0].url : undefined,
        authorAvatar: "authors" in item ? item.authors?.[0].avatar : undefined,
        publishedAt: new Date(getPublishedAt(item.date_published) || (now - index)),
        categories: item.tags,
        media: item.image ?
            [{
              url: item.image,
              type: "photo" as const,
            }] :
          undefined,
        attachments: item.attachments ?
            item.attachments.map((attachment) => ({
              url: attachment.url,
              duration_in_seconds: normalizeDurationInSeconds(attachment.duration_in_seconds),
              mime_type: attachment.mime_type,
              size_in_bytes: attachment.size_in_bytes,
              title: attachment.title,
            })) :
          undefined,
        extra: item._extra ?
            {
              links: item._extra.links?.map((link) => ({
                url: link.url,
                type: link.type,
                content_html: link.content_html,
              })),
            } :
          undefined,
      }
    })

  return {
    feed,
    entries,
  }
}

Convert from RSS

import Parser from "rss-parser"

import { getPublishedAt, safeTTL } from "@/lib/helpers"

import type { FeedModelizerFunction } from "./types"

const parser = new Parser<{
  follow_challenge?: {
    userId: string[]
    feedId: string[]
  }
  ttl?: string
  // this can be a object like { $: { type: 'html' } }
  subtitle?: string | object
  image?: {
    link?: string | string[]
    url: string | string[]
    title?: string | string[]
  }
}>({
  customFields: {
    feed: ["follow_challenge", "subtitle", "image"],
  },
})

/**
 * Convert RSS Feed to DB model
 */
export const modelizeRssFeed: FeedModelizerFunction = async (data: string) => {
  const parsed = await parser.parseString(data)

  const intervalInMin = safeTTL(
    parsed.ttl ? Number.parseInt(parsed.ttl) : undefined,
  )

  const now = Date.now()

  const feed = {
    title: parsed.title,
    description: parsed.description || (typeof parsed.subtitle === "string" ? parsed.subtitle : ""),
    siteUrl: parsed.link,
    image: Array.isArray(parsed.image?.url) ? parsed.image.url[0] : parsed.image?.url,
    checkedAt: new Date(now),
    ttl: intervalInMin,
  }
  if (
    parsed.follow_challenge &&
    parsed.follow_challenge.userId?.at(0) &&
    parsed.follow_challenge.feedId?.at(0)
  ) {
    ;(feed as any)["follow_challenge"] = {
      userId: parsed.follow_challenge.userId[0],
      feedId: parsed.follow_challenge.feedId[0],
    }
  }

  const entries = parsed.items.map((item, index) => {
    item.guid = item.guid || item.id || item.link || item.title
    item.content = item["content:encoded"]?.trim() || item.content?.trim() || item.summary?.trim()

    let author: string | undefined
    if (typeof item.creator === "object") {
      if (Array.isArray((item.creator as any).name)) {
        author = (item.creator as any).name.join(", ")
      } else {
        author = (item.creator as any).name || JSON.stringify(item.creator)
      }
    } else {
      author = item.creator
    }

    return {
      title: item.title,
      url: item.link,
      guid: item.guid!,
      content: item.content,
      author,
      publishedAt: new Date(getPublishedAt(item.pubDate || item.isoDate) || now - index),
      categories: item.categories?.map((category: any) => {
        if (category._) {
          return category._
        } else if (typeof category === "object") {
          return JSON.stringify(category)
        }
        return category
      }),
      attachments: item.enclosure?.url ?
          [
            {
              url: item.enclosure.url,
              mime_type: item.enclosure.type,
              title: undefined,
              size_in_bytes: item.enclosure.length,
              duration_in_seconds: item.itunes?.duration,
            },
          ] :
        undefined,
      media:
        item.enclosure && item.enclosure.type?.startsWith("image/") ?
            [
              {
                url: item.enclosure.url,
                type: "photo" as const,
              },
            ] :
          item.itunes?.image ?
              [
                {
                  url: item.itunes.image,
                  type: "photo" as const,
                },
              ] :
            undefined,
    }
  })

  return {
    feed,
    entries,
  }
}

Convert from Email

import { z } from "zod"

import type { MediaModel } from "@/schema"

import { normalizeHtml } from "./normalize-html"

export const emailZodSchema = z.object({
  from: z.object({
    name: z.string().optional().optional(),
    address: z.string().email().optional(),
  }),
  to: z.object({
    address: z.string().email(),
  }),
  subject: z.string().optional(),
  messageId: z.string(),
  date: z.string().datetime(),
  html: z.string().optional(),
})

export const modelizeEmail = async (email: z.infer<typeof emailZodSchema>, category: string) => {
  let media: MediaModel[] | undefined
  let description: string | undefined

  if (email.html) {
    const nHtml = normalizeHtml(email.html)
    media = nHtml.metadata.media
    description = nHtml.metadata.description
    email.html = nHtml.toHTML()
  }

  return {
    entry: {
      title: email.subject,
      content: email.html,
      description,
      guid: email.messageId,
      author: email.from.name,
      authorUrl: email.from.address && `mailto:${email.from.address}`,
      insertedAt: new Date(),
      publishedAt: new Date(email.date),
      categories: [category],
      media,
    },
  }
}

Convert from Webhook

import type { z } from "zod"

import type { MediaModel } from "@/schema"
import { inboxesEntriesInsertOpenAPISchema } from "@/schema"

import { normalizeHtml } from "./normalize-html"

export const payloadZodSchema = inboxesEntriesInsertOpenAPISchema

export const modelizeWebhook = async (payload: z.infer<typeof payloadZodSchema>, category: string) => {
  let media: MediaModel[] | undefined
  let description: string | undefined

  if (payload.content) {
    const nHtml = normalizeHtml(payload.content)
    media = nHtml.metadata.media
    description = nHtml.metadata.description
    payload.content = nHtml.toHTML()
  }

  return {
    entry: {
      ...payload,
      publishedAt: new Date(payload.publishedAt),
      insertedAt: new Date(),
      categories: [...(payload.categories || []), category],
      media: payload.media || media,
      description: payload.description || description,
    },
  }
}