Caching feeds, minimizing HTTP requests.

components/SeriesCard.tsx

@@ -1,5 +1,5 @@
-import { SearchResult } from "../lib/nrk.ts";
 import CopyButton from "../islands/CopyButton.tsx";
+import { SearchResult } from "../lib/nrk/nrk.ts";
 
 export default function SeriesCard(props: { serie: SearchResult; origin: string }) {
   const feedUrl = new URL(`/api/feeds/${props.serie.seriesId}`, props.origin);

deno.json

@@ -8,6 +8,9 @@
     "check": "deno fmt --check && deno lint && deno check **/*.ts && deno check **/*.tsx",
     "types:nrk": "deno run -A generate-types.ts && deno fmt ./lib"
   },
+  "unstable": [
+    "kv"
+  ],
   "imports": {
     "$fresh/": "https://deno.land/x/fresh@1.6.8/",
     "preact": "https://esm.sh/preact@10.19.6",
@@ -16,7 +19,10 @@
     "@preact/signals": "https://esm.sh/*@preact/signals@1.2.2",
     "@preact/signals-core": "https://esm.sh/*@preact/signals-core@1.5.1",
     "$gfm": "https://deno.land/x/gfm@0.6.0/mod.ts",
-    "openapi-typescript": "npm:openapi-typescript@6.7.5"
+    "openapi-typescript": "npm:openapi-typescript@6.7.5",
+    "datetime": "https://deno.land/std@0.221.0/datetime/mod.ts",
+    "asserts": "https://deno.land/std@0.221.0/assert/mod.ts",
+    "serialize-xml": "https://raw.githubusercontent.com/olaven/serialize-xml/v0.4.0/mod.ts"
   },
   "compilerOptions": {
     "jsx": "react-jsx",

lib/caching.ts

@@ -0,0 +1,78 @@
+import { nrkRadio } from "./nrk/nrk.ts";
+import { Series, storage } from "./storage.ts";
+import * as datetime from "datetime";
+
+const SYNC_INTERVAL_HOURS = 1;
+
+async function initialFetch(options: { id: string }) {
+  const series = await nrkRadio.getSeries(options.id);
+  const stored = storage.write(series);
+  if (!stored) {
+    throw new Error(`Failed to store series ${options.id}`);
+  }
+
+  return series;
+}
+
+async function updateFetch(existingSeries: Series) {
+  const series = await nrkRadio.getSeries(existingSeries.id);
+  const newEpisodes = series.episodes.filter((episode) => {
+    return !existingSeries.episodes.find((serieEpisode) => serieEpisode.id === episode.id);
+  });
+
+  const updated = {
+    ...existingSeries,
+    lastFetch: new Date(),
+    /**
+     * Since we don't control the API,
+     * we should not make assumptions about the order,
+     * but rather sort the episode to what we want.
+     */
+    episodes: [...newEpisodes, ...existingSeries.episodes]
+      .sort((a, b) => a.date.getTime() > b.date.getTime() ? -1 : 1),
+  };
+
+  const updateSuccessful = await storage.write(updated);
+  if (!updateSuccessful) {
+    throw new Error(`Failed to update series ${existingSeries.id}`);
+  }
+
+  return updated;
+}
+
+async function getSeries(options: { id: string }): Promise<Series> {
+  const seriesFromStorage = await storage.read(options);
+
+  /**
+   * We don't have the feed in storage,
+   * and we need to fetch it for the first time.
+   */
+  if (seriesFromStorage === null) {
+    // TODO: schedule a job to fetch the entire backlog https://github.com/olaven/NRSS/issues/24
+    return await initialFetch(options);
+  }
+
+  const timeSinceLastFetch =
+    datetime.difference(seriesFromStorage.lastFetchedAt, new Date(), { units: ["hours"] }).hours;
+
+  // we have the feed in storage and it's not too old
+  if (
+    seriesFromStorage !== null &&
+    timeSinceLastFetch !== null &&
+    timeSinceLastFetch !== undefined &&
+    timeSinceLastFetch <= SYNC_INTERVAL_HOURS
+  ) {
+    return seriesFromStorage;
+  }
+
+  /**
+   * We have the feed in storage, but it's too old
+   * and needs to be refreshed.
+   */
+  const updatedSeries = await updateFetch(seriesFromStorage);
+  return updatedSeries;
+}
+
+export const caching = {
+  getSeries,
+};

lib/nrk/nrk.ts

@@ -2,28 +2,54 @@ import { get, OK } from "https://deno.land/x/kall@v0.1.0/mod.ts";
 import { components as searchComponents } from "./nrk-search.ts";
 import { components as catalogComponents } from "./nrk-catalog.ts";
 import { external as playbackComponents } from "./nrk-playback.ts";
+import { Series } from "../storage.ts";
 
 type ArrayElement<A> = A extends readonly (infer T)[] ? T : never;
 type PodcastEpisodes = catalogComponents["schemas"]["EpisodesHalResource"];
 type Podcast = catalogComponents["schemas"]["SeriesHalResource"];
 type PodcastEpisodesSingle = catalogComponents["schemas"]["EpisodeHalResource"];
 
-export type Serie = catalogComponents["schemas"]["SeriesViewModel"];
+export type NrkSerie = catalogComponents["schemas"]["SeriesViewModel"];
-export type OriginalEpisode = PodcastEpisodesSingle & { url: string };
+export type NrkOriginalEpisode = PodcastEpisodesSingle & { url: string };
-export type PodcastEpisode = catalogComponents["schemas"]["PodcastEpisodeHalResource"];
+export type NrkPodcastEpisode = catalogComponents["schemas"]["PodcastEpisodeHalResource"];
-export type SearchResultList = searchComponents["schemas"]["seriesResult"]["results"];
+export type NrkSearchResultList = searchComponents["schemas"]["seriesResult"]["results"];
-export type SearchResult = ArrayElement<SearchResultList> & {
+export type SearchResult = ArrayElement<NrkSearchResultList> & {
   description?: string;
 };
 
 type Manifest = playbackComponents["schemas/playback-channel.json"]["components"]["schemas"]["PlayableManifest"];
+type NrkSerieData = Awaited<ReturnType<typeof nrkRadio.getSeries>>;
+
+function parseSeries(nrkSeries: NrkSerieData): Series {
+  const imageUrl = nrkSeries.squareImage?.at(-1)?.url ?? "";
+
+  return {
+    id: nrkSeries.id,
+    title: nrkSeries.titles.title,
+    subtitle: nrkSeries.titles.subtitle ?? null,
+    link: `https://radio.nrk.no/podkast/${nrkSeries.id}`,
+    imageUrl: imageUrl,
+    lastFetchedAt: new Date(),
+    episodes: nrkSeries.episodes.map((episode) => {
+      return {
+        id: episode.id,
+        title: episode.titles.title,
+        subtitle: episode.titles.subtitle ?? null,
+        url: episode.url,
+        shareLink: episode._links.share?.href ?? "",
+        date: new Date(episode.date),
+        durationInSeconds: episode.durationInSeconds,
+      };
+    }),
+  };
+}
 
 const nrkAPI = `https://psapi.nrk.no`;
 
 async function withDownloadLink(
   episode: PodcastEpisodesSingle,
   type: catalogComponents["schemas"]["Type"],
-): Promise<OriginalEpisode> {
+): Promise<NrkOriginalEpisode> {
   // getting stream link
   let [playbackStatus, playbackResponse] = await get<Manifest>(
     `${nrkAPI}/playback/manifest/podcast/${episode.episodeId}`,
@@ -43,7 +69,7 @@ async function withDownloadLink(
 }
 
 export const nrkRadio = {
-  search: async (query: string): Promise<SearchResultList> => {
+  search: async (query: string): Promise<NrkSearchResultList> => {
     const [status, response] = await get<
       searchComponents["schemas"]["searchresult"]
     >(`${nrkAPI}/radio/search/search?q=${query}`);
@@ -52,7 +78,7 @@ export const nrkRadio = {
     }
     throw `Something went wrong with ${query} - got status ${status}`;
   },
-  getSerieData: async (seriesId: string) => {
+  getSeries: async (seriesId: string) => {
     let [
       [episodeStatus, episodeResponse],
       [seriesStatus, serieResponse],
@@ -86,16 +112,20 @@ export const nrkRadio = {
       const episodes = await Promise.all(
         episodeResponse._embedded.episodes.map((episode) => withDownloadLink(episode, serieResponse.type)),
       );
-      return {
+
+      const serieData = {
         ...serieResponse.series,
         episodes,
       };
+
+      const parsedSeries = parseSeries(serieData);
+      return parsedSeries;
     }
     throw `Error getting episodes for ${seriesId}: EpisodeStatus: ${episodeStatus}. SerieStatus: ${seriesStatus}`;
   },
-  getEpisode: async (seriesId: string, episodeId: string): Promise<PodcastEpisode | null> => {
+  getEpisode: async (seriesId: string, episodeId: string): Promise<NrkPodcastEpisode | null> => {
     const url = `${nrkAPI}/radio/catalog/podcast/${seriesId}/episodes/${episodeId}`;
-    const [status, episode] = await get<PodcastEpisode>(url);
+    const [status, episode] = await get<NrkPodcastEpisode>(url);
     if (status === OK && episode) {
       return episode;
     }

lib/rss.test.ts

@@ -0,0 +1,19 @@
+import { assertEquals, assertExists } from "asserts";
+import { testUtils } from "./test-utils.ts";
+import { rss } from "./rss.ts";
+
+Deno.test("generated rss contains the series title", () => {
+  const series = testUtils.generateSeries();
+  const feed = rss.assembleFeed(series);
+  assertExists(feed);
+  assertEquals(feed.includes(series.title), true);
+});
+
+Deno.test("generated rss contains all episode titles", () => {
+  const series = testUtils.generateSeries();
+  const feed = rss.assembleFeed(series);
+  assertExists(feed);
+  series.episodes.forEach((episode) => {
+    assertEquals(feed.includes(episode.title), true);
+  });
+});

lib/rss.ts

@@ -0,0 +1,90 @@
+import { declaration, serialize, tag } from "serialize-xml";
+import { getHostName } from "../utils.ts";
+import { Episode, Series } from "./storage.ts";
+
+function assembleFeed(series: Series) {
+  // Originally adapted from https://raw.githubusercontent.com/olaven/paperpod/1cde9abd3174b26e126aa74fc5a3b63fd078c0fd/packages/converter/src/rss.ts
+  return serialize(
+    declaration([
+      ["version", "1.0"],
+      ["encoding", "UTF-8"],
+    ]),
+    tag(
+      "rss",
+      [
+        tag("channel", [
+          tag("title", series.title),
+          tag("link", series.link),
+          tag("itunes:author", "NRK"),
+          /**
+           * serie.category.id does not overlap with Apple's supported categories..
+           * These podcast feeds are not going to be indexed in itunes anyways, so
+           * a static, valid category is fine. The point is simply to pass third party
+           * podcast feed validation.
+           */
+          tag("itunes:category", "", [["text", "Government"]]),
+          tag("itunes:owner", [
+            tag("itunes:name", "NRK"),
+            tag("itunes:email", "nrkpodcast@nrk.no"),
+          ]),
+          tag(
+            "description",
+            series.subtitle || "",
+          ),
+          tag("ttl", "60"), //60 minutes
+          ...(series.imageUrl
+            ? [
+              tag("itunes:image", "", [
+                ["href", series.imageUrl],
+              ]),
+              tag("image", [
+                tag("url", series.imageUrl),
+                tag("title", series.title),
+                tag("link", series.link),
+              ]),
+            ]
+            : []),
+          ...series.episodes.map((episode) => assembleEpisode({ series, episode })),
+        ]),
+      ],
+      [
+        ["version", "2.0"],
+        ["xmlns:itunes", "http://www.itunes.com/dtds/podcast-1.0.dtd"],
+        ["xmlns:content", "http://purl.org/rss/1.0/modules/content/"],
+        ["xmlns:podcast", "https://podcastindex.org/namespace/1.0"],
+      ],
+    ),
+  );
+}
+
+function assembleEpisode(options: { series: Series; episode: Episode }) {
+  const { series, episode } = options;
+
+  const description = episode.subtitle || "";
+
+  return tag("item", [
+    tag("title", episode.title),
+    tag("link", episode.shareLink),
+    tag("description", description),
+    tag("itunes:summary", description),
+    tag("guid", episode.id, [["isPermaLink", "false"]]),
+    tag("pubDate", new Date(episode.date).toUTCString()),
+    tag("itunes:duration", episode.durationInSeconds.toString()),
+    tag("podcast:chapters", "", [
+      [
+        "url",
+        `${getHostName()}/api/feeds/${series.id}/${episode.id}/chapters`,
+      ],
+      ["type", "application/json+chapters"],
+    ]),
+    tag("enclosure", "", [
+      ["url", episode.url],
+      ["length", episode.durationInSeconds.toString()],
+      ["type", "audio/mpeg3"],
+    ]),
+  ]);
+}
+
+export const rss = {
+  assembleFeed,
+};

lib/storage.test.ts

@@ -0,0 +1,21 @@
+import { assertEquals } from "asserts";
+import { storage } from "./storage.ts";
+import { testUtils } from "./test-utils.ts";
+
+Deno.test("can store a series", async () => {
+  const series = testUtils.generateSeries();
+  await storage.write(series);
+
+  const readSeries = await storage.read(series);
+
+  assertEquals(readSeries, series);
+});
+
+Deno.test("can retrieve a series", async () => {
+  const series = testUtils.generateSeries();
+  await storage.write(series);
+
+  const readSeries = await storage.read(series);
+
+  assertEquals(readSeries, series);
+});

lib/storage.ts

@@ -0,0 +1,43 @@
+export type Episode = {
+  id: string;
+  title: string;
+  subtitle: string | null;
+  url: string;
+  shareLink: string;
+  date: Date;
+  durationInSeconds: number;
+};
+
+export type Series = {
+  id: string;
+  title: string;
+  subtitle: string | null;
+  link: string;
+  imageUrl: string;
+  lastFetchedAt: Date;
+  episodes: Episode[];
+};
+
+const kv = await Deno.openKv();
+
+function seriesKey(series: { id: string }) {
+  return ["series", series.id];
+}
+
+async function read(options: { id: string }): Promise<Series | null> {
+  const { id } = options;
+  const key = seriesKey({ id });
+  const read = await kv.get<Series>(key);
+  return read.value;
+}
+
+async function write(series: Series): Promise<boolean> {
+  const key = seriesKey(series);
+  const stored = await kv.set(key, series);
+  return stored.ok;
+}
+
+export const storage = {
+  read,
+  write,
+};

lib/test-utils.ts

@@ -0,0 +1,31 @@
+import { fakerNB_NO as faker } from "npm:@faker-js/faker";
+import { Series } from "./storage.ts";
+
+function generateSeries(overrides: Partial<Series> = {}): Series {
+  return {
+    id: faker.word.words(1),
+    title: faker.word.words(3),
+    subtitle: faker.word.words(3),
+    link: faker.internet.url(),
+    imageUrl: faker.image.url(),
+    lastFetchedAt: faker.date.recent(),
+    episodes: new Array(faker.number.int({ min: 0, max: 100 }))
+      .fill(null).map(() => ({
+        title: faker.word.words(3),
+        subtitle: faker.word.words(3),
+        link: faker.internet.url(),
+        description: faker.lorem.paragraphs(2),
+        id: faker.string.uuid(),
+        date: faker.date.recent(),
+        durationInSeconds: faker.number.int({
+          min: 0,
+          max: 3600,
+        }),
+        ...overrides,
+      })),
+  };
+}
+
+export const testUtils = {
+  generateSeries,
+};

routes/api/feeds/[seriesId].ts

@@ -1,108 +1,16 @@
-import { nrkRadio, OriginalEpisode } from "../../../lib/nrk.ts";
 import { FreshContext } from "$fresh/server.ts";
-import { declaration, serialize, tag } from "https://raw.githubusercontent.com/olaven/serialize-xml/v0.4.0/mod.ts";
+import { caching } from "../../../lib/caching.ts";
-import { getHostName } from "../../../utils.ts";
+import { rss } from "../../../lib/rss.ts";
-
-function toItemTag(seriesId: string, episode: OriginalEpisode) {
-  const description = episode.titles.subtitle || "";
-  return tag("item", [
-    tag("title", episode.titles.title),
-    tag("link", episode._links.share?.href),
-    tag("description", description),
-    tag("itunes:summary", description),
-    tag("guid", episode.id, [["isPermaLink", "false"]]),
-    tag("pubDate", new Date(episode.date).toUTCString()),
-    tag("itunes:duration", episode.durationInSeconds.toString()),
-    tag("podcast:chapters", "", [
-      [
-        "url",
-        `${getHostName()}/api/feeds/${seriesId}/${episode.episodeId}/chapters`,
-      ],
-      ["type", "application/json+chapters"],
-    ]),
-    tag("enclosure", "", [
-      ["url", episode.url],
-      ["length", episode.durationInSeconds.toString()],
-      ["type", "audio/mpeg3"],
-    ]),
-  ]);
-}
-
-async function buildFeed(seriesId: string) {
-  const serie = await nrkRadio.getSerieData(seriesId);
-  const imageUrl = serie.squareImage?.at(-1)?.url ?? "";
-  const linkValue = `https://radio.nrk.no/podkast/${serie.id}`;
-
-  // Quickly adapted from https://raw.githubusercontent.com/olaven/paperpod/1cde9abd3174b26e126aa74fc5a3b63fd078c0fd/packages/converter/src/rss.ts
-  return serialize(
-    declaration([
-      ["version", "1.0"],
-      ["encoding", "UTF-8"],
-    ]),
-    tag(
-      "rss",
-      [
-        tag("channel", [
-          tag("title", serie.titles.title),
-          tag("link", linkValue),
-          tag("itunes:author", "NRK"),
-          /* serie.category.id does not overlap with Apple's supported categories..
-                       These podcast feeds are not going to be indexed in itunes anyways, so
-                       a static, valid category is fine. The point is simply to pass third party
-                       podcast feed validation.
-                       */
-          tag("itunes:category", "", [["text", "Government"]]),
-          tag("itunes:owner", [
-            tag("itunes:name", "NRK"),
-            tag("itunes:email", "nrkpodcast@nrk.no"),
-          ]),
-          tag(
-            "description",
-            serie.titles.subtitle || "",
-          ),
-          tag("ttl", "60"), //60 minutes
-          ...(imageUrl
-            ? [
-              tag("itunes:image", "", [
-                ["href", imageUrl],
-              ]),
-              tag("image", [
-                tag("url", imageUrl),
-                tag("title", serie.titles.title),
-                tag("link", linkValue),
-              ]),
-            ]
-            : []),
-          ...serie.episodes.map((episode) => toItemTag(seriesId, episode)),
-        ]),
-      ],
-      [
-        ["version", "2.0"],
-        ["xmlns:itunes", "http://www.itunes.com/dtds/podcast-1.0.dtd"],
-        ["xmlns:content", "http://purl.org/rss/1.0/modules/content/"],
-        ["xmlns:podcast", "https://podcastindex.org/namespace/1.0"],
-      ],
-    ),
-  );
-}
 
 export const handler = async (_req: Request, ctx: FreshContext): Promise<Response> => {
   const seriesId = ctx.params.seriesId;
-  try {
-    const feedContent = await buildFeed(seriesId);
 
-    return new Response(feedContent, {
+  const series = await caching.getSeries({ id: seriesId });
-      headers: {
+  const feed = rss.assembleFeed(series);
-        "Content-Type": "application/xml",
+
-      },
+  return new Response(feed, {
-    });
+    headers: {
-  } catch (error) {
+      "Content-Type": "application/xml",
-    console.error(error);
+    },
-    return new Response(JSON.stringify({ message: "Could not generate feed" }), {
+  });
-      status: 500,
-      headers: {
-        "Content-Type": "application/json",
-      },
-    });
-  }
 };

routes/api/feeds/[seriesId]/[episodeId]/chapters.ts

@@ -1,13 +1,13 @@
 import { FreshContext } from "$fresh/server.ts";
 import { parse, toSeconds } from "https://esm.sh/iso8601-duration@2.1.1";
-import { nrkRadio, PodcastEpisode } from "../../../../../lib/nrk.ts";
+import { NrkPodcastEpisode, nrkRadio } from "../../../../../lib/nrk/nrk.ts";
 
 type Chapter = {
   title: string | undefined;
   startTime: number | undefined;
 };
 
-function toChapters(episode: PodcastEpisode): Chapter[] | null {
+function toChapters(episode: NrkPodcastEpisode): Chapter[] | null {
   if (!episode.indexPoints) {
     return null;
   }

routes/index.tsx

@@ -4,21 +4,21 @@ import Footer from "../components/Footer.tsx";
 import Header from "../components/Header.tsx";
 import Search from "../components/Search.tsx";
 import SeriesCard from "../components/SeriesCard.tsx";
-import { nrkRadio, SearchResultList } from "../lib/nrk.ts";
 import { CSS, render } from "$gfm";
+import { nrkRadio, NrkSearchResultList } from "../lib/nrk/nrk.ts";
 
 interface HandlerData {
   query: string;
   origin: string;
   rawMarkdown: string;
-  result?: SearchResultList;
+  result?: NrkSearchResultList;
 }
 
 export const handler: Handlers<HandlerData> = {
   async GET(request, ctx) {
     const url = new URL(request.url);
     const query = url.searchParams.get("query");
-    let result: SearchResultList | undefined;
+    let result: NrkSearchResultList | undefined;
     if (query) {
       result = await nrkRadio.search(query);
     }
@@ -57,7 +57,7 @@ export default function Home({ data }: PageProps<HandlerData>) {
   );
 }
 
-function SearchResult({ result, origin }: { result: SearchResultList; origin: string }) {
+function SearchResult({ result, origin }: { result: NrkSearchResultList; origin: string }) {
   if (!result) {
     return null;
   }