From 95d4ecd9749fede353bac42f9b9ee8cf960756ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Olav=20Sundf=C3=B8r?= Date: Sun, 7 Apr 2024 19:15:52 +0200 Subject: [PATCH] Caching feeds, minimizing HTTP requests. --- components/SeriesCard.tsx | 2 +- deno.json | 8 +- lib/caching.ts | 78 ++++++++++++ lib/{ => nrk}/nrk-catalog.ts | 0 lib/{ => nrk}/nrk-playback.ts | 0 lib/{ => nrk}/nrk-search.ts | 0 lib/{ => nrk}/nrk.ts | 52 ++++++-- lib/rss.test.ts | 19 +++ lib/rss.ts | 90 ++++++++++++++ lib/storage.test.ts | 21 ++++ lib/storage.ts | 43 +++++++ lib/test-utils.ts | 31 +++++ routes/api/feeds/[seriesId].ts | 112 ++---------------- .../feeds/[seriesId]/[episodeId]/chapters.ts | 4 +- routes/index.tsx | 8 +- 15 files changed, 347 insertions(+), 121 deletions(-) create mode 100644 lib/caching.ts rename lib/{ => nrk}/nrk-catalog.ts (100%) rename lib/{ => nrk}/nrk-playback.ts (100%) rename lib/{ => nrk}/nrk-search.ts (100%) rename lib/{ => nrk}/nrk.ts (68%) create mode 100644 lib/rss.test.ts create mode 100644 lib/rss.ts create mode 100644 lib/storage.test.ts create mode 100644 lib/storage.ts create mode 100644 lib/test-utils.ts diff --git a/components/SeriesCard.tsx b/components/SeriesCard.tsx index 50ed1b9..479c6ee 100644 --- a/components/SeriesCard.tsx +++ b/components/SeriesCard.tsx @@ -1,5 +1,5 @@ -import { SearchResult } from "../lib/nrk.ts"; import CopyButton from "../islands/CopyButton.tsx"; +import { SearchResult } from "../lib/nrk/nrk.ts"; export default function SeriesCard(props: { serie: SearchResult; origin: string }) { const feedUrl = new URL(`/api/feeds/${props.serie.seriesId}`, props.origin); diff --git a/deno.json b/deno.json index e8e12b5..f1f68ed 100644 --- a/deno.json +++ b/deno.json @@ -8,6 +8,9 @@ "check": "deno fmt --check && deno lint && deno check **/*.ts && deno check **/*.tsx", "types:nrk": "deno run -A generate-types.ts && deno fmt ./lib" }, + "unstable": [ + "kv" + ], "imports": { "$fresh/": "https://deno.land/x/fresh@1.6.8/", "preact": "https://esm.sh/preact@10.19.6", @@ -16,7 +19,10 @@ "@preact/signals": "https://esm.sh/*@preact/signals@1.2.2", "@preact/signals-core": "https://esm.sh/*@preact/signals-core@1.5.1", "$gfm": "https://deno.land/x/gfm@0.6.0/mod.ts", - "openapi-typescript": "npm:openapi-typescript@6.7.5" + "openapi-typescript": "npm:openapi-typescript@6.7.5", + "datetime": "https://deno.land/std@0.221.0/datetime/mod.ts", + "asserts": "https://deno.land/std@0.221.0/assert/mod.ts", + "serialize-xml": "https://raw.githubusercontent.com/olaven/serialize-xml/v0.4.0/mod.ts" }, "compilerOptions": { "jsx": "react-jsx", diff --git a/lib/caching.ts b/lib/caching.ts new file mode 100644 index 0000000..67e4ac1 --- /dev/null +++ b/lib/caching.ts @@ -0,0 +1,78 @@ +import { nrkRadio } from "./nrk/nrk.ts"; +import { Series, storage } from "./storage.ts"; +import * as datetime from "datetime"; + +const SYNC_INTERVAL_HOURS = 1; + +async function initialFetch(options: { id: string }) { + const series = await nrkRadio.getSeries(options.id); + const stored = storage.write(series); + if (!stored) { + throw new Error(`Failed to store series ${options.id}`); + } + + return series; +} + +async function updateFetch(existingSeries: Series) { + const series = await nrkRadio.getSeries(existingSeries.id); + const newEpisodes = series.episodes.filter((episode) => { + return !existingSeries.episodes.find((serieEpisode) => serieEpisode.id === episode.id); + }); + + const updated = { + ...existingSeries, + lastFetch: new Date(), + /** + * Since we don't control the API, + * we should not make assumptions about the order, + * but rather sort the episode to what we want. + */ + episodes: [...newEpisodes, ...existingSeries.episodes] + .sort((a, b) => a.date.getTime() > b.date.getTime() ? -1 : 1), + }; + + const updateSuccessful = await storage.write(updated); + if (!updateSuccessful) { + throw new Error(`Failed to update series ${existingSeries.id}`); + } + + return updated; +} + +async function getSeries(options: { id: string }): Promise { + const seriesFromStorage = await storage.read(options); + + /** + * We don't have the feed in storage, + * and we need to fetch it for the first time. + */ + if (seriesFromStorage === null) { + // TODO: schedule a job to fetch the entire backlog https://github.com/olaven/NRSS/issues/24 + return await initialFetch(options); + } + + const timeSinceLastFetch = + datetime.difference(seriesFromStorage.lastFetchedAt, new Date(), { units: ["hours"] }).hours; + + // we have the feed in storage and it's not too old + if ( + seriesFromStorage !== null && + timeSinceLastFetch !== null && + timeSinceLastFetch !== undefined && + timeSinceLastFetch <= SYNC_INTERVAL_HOURS + ) { + return seriesFromStorage; + } + + /** + * We have the feed in storage, but it's too old + * and needs to be refreshed. + */ + const updatedSeries = await updateFetch(seriesFromStorage); + return updatedSeries; +} + +export const caching = { + getSeries, +}; diff --git a/lib/nrk-catalog.ts b/lib/nrk/nrk-catalog.ts similarity index 100% rename from lib/nrk-catalog.ts rename to lib/nrk/nrk-catalog.ts diff --git a/lib/nrk-playback.ts b/lib/nrk/nrk-playback.ts similarity index 100% rename from lib/nrk-playback.ts rename to lib/nrk/nrk-playback.ts diff --git a/lib/nrk-search.ts b/lib/nrk/nrk-search.ts similarity index 100% rename from lib/nrk-search.ts rename to lib/nrk/nrk-search.ts diff --git a/lib/nrk.ts b/lib/nrk/nrk.ts similarity index 68% rename from lib/nrk.ts rename to lib/nrk/nrk.ts index 75328b7..e18af3d 100644 --- a/lib/nrk.ts +++ b/lib/nrk/nrk.ts @@ -2,28 +2,54 @@ import { get, OK } from "https://deno.land/x/kall@v0.1.0/mod.ts"; import { components as searchComponents } from "./nrk-search.ts"; import { components as catalogComponents } from "./nrk-catalog.ts"; import { external as playbackComponents } from "./nrk-playback.ts"; +import { Series } from "../storage.ts"; type ArrayElement = A extends readonly (infer T)[] ? T : never; type PodcastEpisodes = catalogComponents["schemas"]["EpisodesHalResource"]; type Podcast = catalogComponents["schemas"]["SeriesHalResource"]; type PodcastEpisodesSingle = catalogComponents["schemas"]["EpisodeHalResource"]; -export type Serie = catalogComponents["schemas"]["SeriesViewModel"]; -export type OriginalEpisode = PodcastEpisodesSingle & { url: string }; -export type PodcastEpisode = catalogComponents["schemas"]["PodcastEpisodeHalResource"]; -export type SearchResultList = searchComponents["schemas"]["seriesResult"]["results"]; -export type SearchResult = ArrayElement & { +export type NrkSerie = catalogComponents["schemas"]["SeriesViewModel"]; +export type NrkOriginalEpisode = PodcastEpisodesSingle & { url: string }; +export type NrkPodcastEpisode = catalogComponents["schemas"]["PodcastEpisodeHalResource"]; +export type NrkSearchResultList = searchComponents["schemas"]["seriesResult"]["results"]; +export type SearchResult = ArrayElement & { description?: string; }; type Manifest = playbackComponents["schemas/playback-channel.json"]["components"]["schemas"]["PlayableManifest"]; +type NrkSerieData = Awaited>; + +function parseSeries(nrkSeries: NrkSerieData): Series { + const imageUrl = nrkSeries.squareImage?.at(-1)?.url ?? ""; + + return { + id: nrkSeries.id, + title: nrkSeries.titles.title, + subtitle: nrkSeries.titles.subtitle ?? null, + link: `https://radio.nrk.no/podkast/${nrkSeries.id}`, + imageUrl: imageUrl, + lastFetchedAt: new Date(), + episodes: nrkSeries.episodes.map((episode) => { + return { + id: episode.id, + title: episode.titles.title, + subtitle: episode.titles.subtitle ?? null, + url: episode.url, + shareLink: episode._links.share?.href ?? "", + date: new Date(episode.date), + durationInSeconds: episode.durationInSeconds, + }; + }), + }; +} const nrkAPI = `https://psapi.nrk.no`; async function withDownloadLink( episode: PodcastEpisodesSingle, type: catalogComponents["schemas"]["Type"], -): Promise { +): Promise { // getting stream link let [playbackStatus, playbackResponse] = await get( `${nrkAPI}/playback/manifest/podcast/${episode.episodeId}`, @@ -43,7 +69,7 @@ async function withDownloadLink( } export const nrkRadio = { - search: async (query: string): Promise => { + search: async (query: string): Promise => { const [status, response] = await get< searchComponents["schemas"]["searchresult"] >(`${nrkAPI}/radio/search/search?q=${query}`); @@ -52,7 +78,7 @@ export const nrkRadio = { } throw `Something went wrong with ${query} - got status ${status}`; }, - getSerieData: async (seriesId: string) => { + getSeries: async (seriesId: string) => { let [ [episodeStatus, episodeResponse], [seriesStatus, serieResponse], @@ -86,16 +112,20 @@ export const nrkRadio = { const episodes = await Promise.all( episodeResponse._embedded.episodes.map((episode) => withDownloadLink(episode, serieResponse.type)), ); - return { + + const serieData = { ...serieResponse.series, episodes, }; + + const parsedSeries = parseSeries(serieData); + return parsedSeries; } throw `Error getting episodes for ${seriesId}: EpisodeStatus: ${episodeStatus}. SerieStatus: ${seriesStatus}`; }, - getEpisode: async (seriesId: string, episodeId: string): Promise => { + getEpisode: async (seriesId: string, episodeId: string): Promise => { const url = `${nrkAPI}/radio/catalog/podcast/${seriesId}/episodes/${episodeId}`; - const [status, episode] = await get(url); + const [status, episode] = await get(url); if (status === OK && episode) { return episode; } diff --git a/lib/rss.test.ts b/lib/rss.test.ts new file mode 100644 index 0000000..e399023 --- /dev/null +++ b/lib/rss.test.ts @@ -0,0 +1,19 @@ +import { assertEquals, assertExists } from "asserts"; +import { testUtils } from "./test-utils.ts"; +import { rss } from "./rss.ts"; + +Deno.test("generated rss contains the series title", () => { + const series = testUtils.generateSeries(); + const feed = rss.assembleFeed(series); + assertExists(feed); + assertEquals(feed.includes(series.title), true); +}); + +Deno.test("generated rss contains all episode titles", () => { + const series = testUtils.generateSeries(); + const feed = rss.assembleFeed(series); + assertExists(feed); + series.episodes.forEach((episode) => { + assertEquals(feed.includes(episode.title), true); + }); +}); diff --git a/lib/rss.ts b/lib/rss.ts new file mode 100644 index 0000000..7e2a025 --- /dev/null +++ b/lib/rss.ts @@ -0,0 +1,90 @@ +import { declaration, serialize, tag } from "serialize-xml"; +import { getHostName } from "../utils.ts"; +import { Episode, Series } from "./storage.ts"; + +function assembleFeed(series: Series) { + // Originally adapted from https://raw.githubusercontent.com/olaven/paperpod/1cde9abd3174b26e126aa74fc5a3b63fd078c0fd/packages/converter/src/rss.ts + return serialize( + declaration([ + ["version", "1.0"], + ["encoding", "UTF-8"], + ]), + tag( + "rss", + [ + tag("channel", [ + tag("title", series.title), + tag("link", series.link), + tag("itunes:author", "NRK"), + /** + * serie.category.id does not overlap with Apple's supported categories.. + * These podcast feeds are not going to be indexed in itunes anyways, so + * a static, valid category is fine. The point is simply to pass third party + * podcast feed validation. + */ + tag("itunes:category", "", [["text", "Government"]]), + tag("itunes:owner", [ + tag("itunes:name", "NRK"), + tag("itunes:email", "nrkpodcast@nrk.no"), + ]), + tag( + "description", + series.subtitle || "", + ), + tag("ttl", "60"), //60 minutes + ...(series.imageUrl + ? [ + tag("itunes:image", "", [ + ["href", series.imageUrl], + ]), + tag("image", [ + tag("url", series.imageUrl), + tag("title", series.title), + tag("link", series.link), + ]), + ] + : []), + ...series.episodes.map((episode) => assembleEpisode({ series, episode })), + ]), + ], + [ + ["version", "2.0"], + ["xmlns:itunes", "http://www.itunes.com/dtds/podcast-1.0.dtd"], + ["xmlns:content", "http://purl.org/rss/1.0/modules/content/"], + ["xmlns:podcast", "https://podcastindex.org/namespace/1.0"], + ], + ), + ); +} + +function assembleEpisode(options: { series: Series; episode: Episode }) { + const { series, episode } = options; + + const description = episode.subtitle || ""; + + return tag("item", [ + tag("title", episode.title), + tag("link", episode.shareLink), + tag("description", description), + tag("itunes:summary", description), + tag("guid", episode.id, [["isPermaLink", "false"]]), + tag("pubDate", new Date(episode.date).toUTCString()), + tag("itunes:duration", episode.durationInSeconds.toString()), + tag("podcast:chapters", "", [ + [ + "url", + `${getHostName()}/api/feeds/${series.id}/${episode.id}/chapters`, + ], + ["type", "application/json+chapters"], + ]), + tag("enclosure", "", [ + ["url", episode.url], + ["length", episode.durationInSeconds.toString()], + ["type", "audio/mpeg3"], + ]), + ]); +} + +export const rss = { + assembleFeed, +}; diff --git a/lib/storage.test.ts b/lib/storage.test.ts new file mode 100644 index 0000000..f02eedf --- /dev/null +++ b/lib/storage.test.ts @@ -0,0 +1,21 @@ +import { assertEquals } from "asserts"; +import { storage } from "./storage.ts"; +import { testUtils } from "./test-utils.ts"; + +Deno.test("can store a series", async () => { + const series = testUtils.generateSeries(); + await storage.write(series); + + const readSeries = await storage.read(series); + + assertEquals(readSeries, series); +}); + +Deno.test("can retrieve a series", async () => { + const series = testUtils.generateSeries(); + await storage.write(series); + + const readSeries = await storage.read(series); + + assertEquals(readSeries, series); +}); diff --git a/lib/storage.ts b/lib/storage.ts new file mode 100644 index 0000000..6b8e2db --- /dev/null +++ b/lib/storage.ts @@ -0,0 +1,43 @@ +export type Episode = { + id: string; + title: string; + subtitle: string | null; + url: string; + shareLink: string; + date: Date; + durationInSeconds: number; +}; + +export type Series = { + id: string; + title: string; + subtitle: string | null; + link: string; + imageUrl: string; + lastFetchedAt: Date; + episodes: Episode[]; +}; + +const kv = await Deno.openKv(); + +function seriesKey(series: { id: string }) { + return ["series", series.id]; +} + +async function read(options: { id: string }): Promise { + const { id } = options; + const key = seriesKey({ id }); + const read = await kv.get(key); + return read.value; +} + +async function write(series: Series): Promise { + const key = seriesKey(series); + const stored = await kv.set(key, series); + return stored.ok; +} + +export const storage = { + read, + write, +}; diff --git a/lib/test-utils.ts b/lib/test-utils.ts new file mode 100644 index 0000000..8d0ea08 --- /dev/null +++ b/lib/test-utils.ts @@ -0,0 +1,31 @@ +import { fakerNB_NO as faker } from "npm:@faker-js/faker"; +import { Series } from "./storage.ts"; + +function generateSeries(overrides: Partial = {}): Series { + return { + id: faker.word.words(1), + title: faker.word.words(3), + subtitle: faker.word.words(3), + link: faker.internet.url(), + imageUrl: faker.image.url(), + lastFetchedAt: faker.date.recent(), + episodes: new Array(faker.number.int({ min: 0, max: 100 })) + .fill(null).map(() => ({ + title: faker.word.words(3), + subtitle: faker.word.words(3), + link: faker.internet.url(), + description: faker.lorem.paragraphs(2), + id: faker.string.uuid(), + date: faker.date.recent(), + durationInSeconds: faker.number.int({ + min: 0, + max: 3600, + }), + ...overrides, + })), + }; +} + +export const testUtils = { + generateSeries, +}; diff --git a/routes/api/feeds/[seriesId].ts b/routes/api/feeds/[seriesId].ts index 8729249..fa57fdc 100644 --- a/routes/api/feeds/[seriesId].ts +++ b/routes/api/feeds/[seriesId].ts @@ -1,108 +1,16 @@ -import { nrkRadio, OriginalEpisode } from "../../../lib/nrk.ts"; import { FreshContext } from "$fresh/server.ts"; -import { declaration, serialize, tag } from "https://raw.githubusercontent.com/olaven/serialize-xml/v0.4.0/mod.ts"; -import { getHostName } from "../../../utils.ts"; - -function toItemTag(seriesId: string, episode: OriginalEpisode) { - const description = episode.titles.subtitle || ""; - return tag("item", [ - tag("title", episode.titles.title), - tag("link", episode._links.share?.href), - tag("description", description), - tag("itunes:summary", description), - tag("guid", episode.id, [["isPermaLink", "false"]]), - tag("pubDate", new Date(episode.date).toUTCString()), - tag("itunes:duration", episode.durationInSeconds.toString()), - tag("podcast:chapters", "", [ - [ - "url", - `${getHostName()}/api/feeds/${seriesId}/${episode.episodeId}/chapters`, - ], - ["type", "application/json+chapters"], - ]), - tag("enclosure", "", [ - ["url", episode.url], - ["length", episode.durationInSeconds.toString()], - ["type", "audio/mpeg3"], - ]), - ]); -} - -async function buildFeed(seriesId: string) { - const serie = await nrkRadio.getSerieData(seriesId); - const imageUrl = serie.squareImage?.at(-1)?.url ?? ""; - const linkValue = `https://radio.nrk.no/podkast/${serie.id}`; - - // Quickly adapted from https://raw.githubusercontent.com/olaven/paperpod/1cde9abd3174b26e126aa74fc5a3b63fd078c0fd/packages/converter/src/rss.ts - return serialize( - declaration([ - ["version", "1.0"], - ["encoding", "UTF-8"], - ]), - tag( - "rss", - [ - tag("channel", [ - tag("title", serie.titles.title), - tag("link", linkValue), - tag("itunes:author", "NRK"), - /* serie.category.id does not overlap with Apple's supported categories.. - These podcast feeds are not going to be indexed in itunes anyways, so - a static, valid category is fine. The point is simply to pass third party - podcast feed validation. - */ - tag("itunes:category", "", [["text", "Government"]]), - tag("itunes:owner", [ - tag("itunes:name", "NRK"), - tag("itunes:email", "nrkpodcast@nrk.no"), - ]), - tag( - "description", - serie.titles.subtitle || "", - ), - tag("ttl", "60"), //60 minutes - ...(imageUrl - ? [ - tag("itunes:image", "", [ - ["href", imageUrl], - ]), - tag("image", [ - tag("url", imageUrl), - tag("title", serie.titles.title), - tag("link", linkValue), - ]), - ] - : []), - ...serie.episodes.map((episode) => toItemTag(seriesId, episode)), - ]), - ], - [ - ["version", "2.0"], - ["xmlns:itunes", "http://www.itunes.com/dtds/podcast-1.0.dtd"], - ["xmlns:content", "http://purl.org/rss/1.0/modules/content/"], - ["xmlns:podcast", "https://podcastindex.org/namespace/1.0"], - ], - ), - ); -} +import { caching } from "../../../lib/caching.ts"; +import { rss } from "../../../lib/rss.ts"; export const handler = async (_req: Request, ctx: FreshContext): Promise => { const seriesId = ctx.params.seriesId; - try { - const feedContent = await buildFeed(seriesId); - return new Response(feedContent, { - headers: { - "Content-Type": "application/xml", - }, - }); - } catch (error) { - console.error(error); - return new Response(JSON.stringify({ message: "Could not generate feed" }), { - status: 500, - headers: { - "Content-Type": "application/json", - }, - }); - } + const series = await caching.getSeries({ id: seriesId }); + const feed = rss.assembleFeed(series); + + return new Response(feed, { + headers: { + "Content-Type": "application/xml", + }, + }); }; diff --git a/routes/api/feeds/[seriesId]/[episodeId]/chapters.ts b/routes/api/feeds/[seriesId]/[episodeId]/chapters.ts index 19eee90..013fa7f 100644 --- a/routes/api/feeds/[seriesId]/[episodeId]/chapters.ts +++ b/routes/api/feeds/[seriesId]/[episodeId]/chapters.ts @@ -1,13 +1,13 @@ import { FreshContext } from "$fresh/server.ts"; import { parse, toSeconds } from "https://esm.sh/iso8601-duration@2.1.1"; -import { nrkRadio, PodcastEpisode } from "../../../../../lib/nrk.ts"; +import { NrkPodcastEpisode, nrkRadio } from "../../../../../lib/nrk/nrk.ts"; type Chapter = { title: string | undefined; startTime: number | undefined; }; -function toChapters(episode: PodcastEpisode): Chapter[] | null { +function toChapters(episode: NrkPodcastEpisode): Chapter[] | null { if (!episode.indexPoints) { return null; } diff --git a/routes/index.tsx b/routes/index.tsx index 1d04993..30f6828 100644 --- a/routes/index.tsx +++ b/routes/index.tsx @@ -4,21 +4,21 @@ import Footer from "../components/Footer.tsx"; import Header from "../components/Header.tsx"; import Search from "../components/Search.tsx"; import SeriesCard from "../components/SeriesCard.tsx"; -import { nrkRadio, SearchResultList } from "../lib/nrk.ts"; import { CSS, render } from "$gfm"; +import { nrkRadio, NrkSearchResultList } from "../lib/nrk/nrk.ts"; interface HandlerData { query: string; origin: string; rawMarkdown: string; - result?: SearchResultList; + result?: NrkSearchResultList; } export const handler: Handlers = { async GET(request, ctx) { const url = new URL(request.url); const query = url.searchParams.get("query"); - let result: SearchResultList | undefined; + let result: NrkSearchResultList | undefined; if (query) { result = await nrkRadio.search(query); } @@ -57,7 +57,7 @@ export default function Home({ data }: PageProps) { ); } -function SearchResult({ result, origin }: { result: SearchResultList; origin: string }) { +function SearchResult({ result, origin }: { result: NrkSearchResultList; origin: string }) { if (!result) { return null; }