Merge remote-tracking branch 'upstream/main' into feature-testing

Signed-off-by: Tim Hårek Andreassen <tim@harek.no>

components/SeriesCard.tsx

@@ -1,5 +1,5 @@
-import { SearchResult } from "../lib/nrk.ts";
 import CopyButton from "../islands/CopyButton.tsx";
+import { SearchResult } from "../lib/nrk/nrk.ts";
 
 export default function SeriesCard(props: { serie: SearchResult; origin: string }) {
   const feedUrl = new URL(`/api/feeds/${props.serie.seriesId}`, props.origin);

deno.json

@@ -8,6 +8,9 @@
     "check": "deno fmt --check && deno lint && deno check **/*.ts && deno check **/*.tsx",
     "types:nrk": "deno run -A generate-types.ts && deno fmt ./lib"
   },
+  "unstable": [
+    "kv"
+  ],
   "imports": {
     "$fresh/": "https://deno.land/x/fresh@1.6.8/",
     "preact": "https://esm.sh/preact@10.19.6",
@@ -17,6 +20,7 @@
     "@preact/signals-core": "https://esm.sh/*@preact/signals-core@1.5.1",
     "$gfm": "https://deno.land/x/gfm@0.6.0/mod.ts",
     "openapi-typescript": "npm:openapi-typescript@6.7.5",
+    "serialize-xml": "https://raw.githubusercontent.com/olaven/serialize-xml/v0.4.0/mod.ts",
     "$std/": "https://deno.land/std@0.221.0/"
   },
   "compilerOptions": {

lib/caching.ts

@@ -0,0 +1,90 @@
+import { nrkRadio } from "./nrk/nrk.ts";
+import { Series, storage } from "./storage.ts";
+import * as datetime from "$std/datetime/mod.ts";
+import { Episode } from "./storage.ts";
+
+const SYNC_INTERVAL_HOURS = 1;
+
+async function initialFetch(options: { id: string }): Promise<Series | null> {
+  const series = await nrkRadio.getSeries(options.id);
+  if (!series) {
+    return null;
+  }
+  const stored = storage.write(series);
+  if (!stored) {
+    console.error(`Failed to store series ${options.id}`);
+    return null;
+  }
+
+  return series;
+}
+
+type UpdatedSeries = {
+  lastFetch: Date;
+  episodes: Episode[];
+} & Series;
+async function updateFetch(existingSeries: Series): Promise<UpdatedSeries | null> {
+  const series = await nrkRadio.getSeries(existingSeries.id);
+  if (!series) {
+    return null;
+  }
+  const newEpisodes = series.episodes.filter((episode) => {
+    return !existingSeries.episodes.find((serieEpisode) => serieEpisode.id === episode.id);
+  });
+
+  const updated = {
+    ...existingSeries,
+    lastFetch: new Date(),
+    /**
+     * Since we don't control the API,
+     * we should not make assumptions about the order,
+     * but rather sort the episode to what we want.
+     */
+    episodes: [...newEpisodes, ...existingSeries.episodes]
+      .sort((a, b) => a.date.getTime() > b.date.getTime() ? -1 : 1),
+  };
+
+  const updateSuccessful = await storage.write(updated);
+  if (!updateSuccessful) {
+    throw new Error(`Failed to update series ${existingSeries.id}`);
+  }
+
+  return updated;
+}
+
+async function getSeries(options: { id: string }): Promise<Series | null> {
+  const seriesFromStorage = await storage.read(options);
+
+  /**
+   * We don't have the feed in storage,
+   * and we need to fetch it for the first time.
+   */
+  if (seriesFromStorage === null) {
+    // TODO: schedule a job to fetch the entire backlog https://github.com/olaven/NRSS/issues/24
+    return await initialFetch(options);
+  }
+
+  const timeSinceLastFetch =
+    datetime.difference(seriesFromStorage.lastFetchedAt, new Date(), { units: ["hours"] }).hours;
+
+  // we have the feed in storage and it's not too old
+  if (
+    seriesFromStorage !== null &&
+    timeSinceLastFetch !== null &&
+    timeSinceLastFetch !== undefined &&
+    timeSinceLastFetch <= SYNC_INTERVAL_HOURS
+  ) {
+    return seriesFromStorage;
+  }
+
+  /**
+   * We have the feed in storage, but it's too old
+   * and needs to be refreshed.
+   */
+  const updatedSeries = await updateFetch(seriesFromStorage);
+  return updatedSeries;
+}
+
+export const caching = {
+  getSeries,
+};

lib/nrk/nrk.ts

@@ -2,6 +2,7 @@ import { get, STATUS_CODE } from "https://deno.land/x/kall@v2.0.0/mod.ts";
 import { components as searchComponents } from "./nrk-search.ts";
 import { components as catalogComponents } from "./nrk-catalog.ts";
 import { external as playbackComponents } from "./nrk-playback.ts";
+import { Series } from "../storage.ts";
 
 type ArrayElement<A> = A extends readonly (infer T)[] ? T : never;
 type PodcastEpisodes = catalogComponents["schemas"]["EpisodesHalResource"];
@@ -10,17 +11,42 @@ type PodcastEpisodesSingle = catalogComponents["schemas"]["EpisodeHalResource"];
 type RadioSeriesEpisode = catalogComponents["schemas"]["EpisodesHalResource"];
 type RadioSeries = catalogComponents["schemas"]["SeriesHalResource"];
 
-export type Serie = catalogComponents["schemas"]["SeriesViewModel"];
+export type NrkSerie = catalogComponents["schemas"]["SeriesViewModel"];
-export type OriginalEpisode = PodcastEpisodesSingle & { url: string };
+export type NrkOriginalEpisode = PodcastEpisodesSingle & { url: string };
-export type PodcastEpisode = catalogComponents["schemas"]["PodcastEpisodeHalResource"];
+export type NrkPodcastEpisode = catalogComponents["schemas"]["PodcastEpisodeHalResource"];
-export type SearchResultList = searchComponents["schemas"]["seriesResult"]["results"];
+export type NrkSearchResultList = searchComponents["schemas"]["seriesResult"]["results"];
-export type SearchResult = ArrayElement<SearchResultList> & {
+export type SearchResult = ArrayElement<NrkSearchResultList> & {
   description?: string;
 };
+export type SeriesData = { episodes: NrkOriginalEpisode[] } & (RadioSeries["series"] | Podcast["series"]);
+
+function parseSeries(nrkSeries: SeriesData): Series {
+  const imageUrl = nrkSeries.squareImage?.at(-1)?.url ?? "";
+
+  return {
+    id: nrkSeries.id,
+    title: nrkSeries.titles.title,
+    subtitle: nrkSeries.titles.subtitle ?? null,
+    link: `https://radio.nrk.no/podkast/${nrkSeries.id}`,
+    imageUrl: imageUrl,
+    lastFetchedAt: new Date(),
+    episodes: nrkSeries.episodes.map((episode) => {
+      return {
+        id: episode.id,
+        title: episode.titles.title,
+        subtitle: episode.titles.subtitle ?? null,
+        url: episode.url,
+        shareLink: episode._links.share?.href ?? "",
+        date: new Date(episode.date),
+        durationInSeconds: episode.durationInSeconds,
+      };
+    }),
+  };
+}
 
 const nrkAPI = `https://psapi.nrk.no`;
 
-async function search(query: string): Promise<SearchResultList | null> {
+async function search(query: string): Promise<NrkSearchResultList | null> {
   if (query === "") {
     console.error("Empty search query.");
     return null;
@@ -36,8 +62,7 @@ async function search(query: string): Promise<SearchResultList | null> {
   return null;
 }
 
-export type SeriesData = { episodes: OriginalEpisode[] } & (RadioSeries["series"] | Podcast["series"]);
+async function getSeries(seriesId: string): Promise<Series | null> {
-async function getSerieData(seriesId: string): Promise<SeriesData | null> {
   let [
     { status: episodeStatus, body: episodeResponse },
     { status: seriesStatus, body: serieResponse },
@@ -71,10 +96,12 @@ async function getSerieData(seriesId: string): Promise<SeriesData | null> {
     const episodes = await Promise.all(
       episodeResponse._embedded.episodes.map((episode) => getEpisodeWithDownloadLink(episode, serieResponse.type)),
     );
-    return {
+    const seriesData = {
       ...serieResponse.series,
       episodes,
     };
+    const parsedSeries = parseSeries(seriesData);
+    return parsedSeries;
   }
   console.error(
     `Error getting episodes for ${seriesId}: EpisodeStatus: ${episodeStatus}. SerieStatus: ${seriesStatus}`,
@@ -82,9 +109,9 @@ async function getSerieData(seriesId: string): Promise<SeriesData | null> {
   return null;
 }
 
-async function getEpisode(seriesId: string, episodeId: string): Promise<PodcastEpisode | null> {
+async function getEpisode(seriesId: string, episodeId: string): Promise<NrkPodcastEpisode | null> {
   const url = `${nrkAPI}/radio/catalog/podcast/${seriesId}/episodes/${episodeId}`;
-  const { status, body: episode } = await get<PodcastEpisode>(url);
+  const { status, body: episode } = await get<NrkPodcastEpisode>(url);
   if (status === STATUS_CODE.OK && episode) {
     return episode;
   }
@@ -97,7 +124,7 @@ type Manifest = playbackComponents["schemas/playback-channel.json"]["components"
 async function getEpisodeWithDownloadLink(
   episode: PodcastEpisodesSingle,
   type: catalogComponents["schemas"]["Type"],
-): Promise<OriginalEpisode> {
+): Promise<NrkOriginalEpisode> {
   // getting stream link
   let { status: playbackStatus, body: playbackResponse } = await get<Manifest>(
     `${nrkAPI}/playback/manifest/podcast/${episode.episodeId}`,
@@ -120,10 +147,7 @@ async function getEpisodeWithDownloadLink(
 
 export const nrkRadio = {
   search,
-  getSerieData,
+  getSeries,
   getEpisode,
-};
+  parseSeries,
-
-export const forTestingOnly = {
-  getEpisodeWithDownloadLink,
 };

lib/nrk/nrk_test.ts

@@ -2,7 +2,6 @@ import { assertEquals } from "$std/assert/assert_equals.ts";
 import { nrkRadio } from "./nrk.ts";
 import { assertGreaterOrEqual } from "$std/assert/assert_greater_or_equal.ts";
 import { assertExists } from "https://deno.land/std@0.216.0/assert/assert_exists.ts";
-import { forTestingOnly } from "./nrk.ts";
 
 Deno.test("Verify search query `trygd` returns one result: 'Trygdekontoret'", async () => {
   const result = await nrkRadio.search("trygd");
@@ -18,14 +17,14 @@ Deno.test("Verify empty search query yields `null`", async () => {
 });
 
 Deno.test("Verify getting series data for 'trygdekontoret' works", async () => {
-  const result = await nrkRadio.getSerieData("trygdekontoret");
+  const result = await nrkRadio.getSeries("trygdekontoret");
 
   assertExists(result);
   assertGreaterOrEqual(result.episodes.length, 1);
 });
 
 Deno.test("Verify getting series data for 'trygd' does not works", async () => {
-  const result = await nrkRadio.getSerieData("trygd");
+  const result = await nrkRadio.getSeries("trygd");
   assertEquals(result, null);
 });
 
@@ -41,11 +40,3 @@ Deno.test("Verify getting episodeId 'null' for 'trygdekontoret' yields `null`",
 
   assertEquals(result, null);
 });
-
-Deno.test("Verify getting episode, 'l_0bc5e55a-46b5-48a5-85e5-5a46b5d8a562' with download link works`", async () => {
-  const series = await nrkRadio.getSerieData("trygdekontoret");
-  assertExists(series);
-
-  const result = await forTestingOnly.getEpisodeWithDownloadLink(series.episodes[0], "podcast");
-  assertExists(result);
-});

lib/rss.test.ts

@@ -0,0 +1,19 @@
+import { assertEquals, assertExists } from "$std/assert/mod.ts";
+import { testUtils } from "./test-utils.ts";
+import { rss } from "./rss.ts";
+
+Deno.test("generated rss contains the series title", () => {
+  const series = testUtils.generateSeries();
+  const feed = rss.assembleFeed(series);
+  assertExists(feed);
+  assertEquals(feed.includes(series.title), true);
+});
+
+Deno.test("generated rss contains all episode titles", () => {
+  const series = testUtils.generateSeries();
+  const feed = rss.assembleFeed(series);
+  assertExists(feed);
+  series.episodes.forEach((episode) => {
+    assertEquals(feed.includes(episode.title), true);
+  });
+});

lib/rss.ts

@@ -0,0 +1,90 @@
+import { declaration, serialize, tag } from "serialize-xml";
+import { getHostName } from "../utils.ts";
+import { Episode, Series } from "./storage.ts";
+
+function assembleFeed(series: Series) {
+  // Originally adapted from https://raw.githubusercontent.com/olaven/paperpod/1cde9abd3174b26e126aa74fc5a3b63fd078c0fd/packages/converter/src/rss.ts
+  return serialize(
+    declaration([
+      ["version", "1.0"],
+      ["encoding", "UTF-8"],
+    ]),
+    tag(
+      "rss",
+      [
+        tag("channel", [
+          tag("title", series.title),
+          tag("link", series.link),
+          tag("itunes:author", "NRK"),
+          /**
+           * serie.category.id does not overlap with Apple's supported categories..
+           * These podcast feeds are not going to be indexed in itunes anyways, so
+           * a static, valid category is fine. The point is simply to pass third party
+           * podcast feed validation.
+           */
+          tag("itunes:category", "", [["text", "Government"]]),
+          tag("itunes:owner", [
+            tag("itunes:name", "NRK"),
+            tag("itunes:email", "nrkpodcast@nrk.no"),
+          ]),
+          tag(
+            "description",
+            series.subtitle || "",
+          ),
+          tag("ttl", "60"), //60 minutes
+          ...(series.imageUrl
+            ? [
+              tag("itunes:image", "", [
+                ["href", series.imageUrl],
+              ]),
+              tag("image", [
+                tag("url", series.imageUrl),
+                tag("title", series.title),
+                tag("link", series.link),
+              ]),
+            ]
+            : []),
+          ...series.episodes.map((episode) => assembleEpisode({ series, episode })),
+        ]),
+      ],
+      [
+        ["version", "2.0"],
+        ["xmlns:itunes", "http://www.itunes.com/dtds/podcast-1.0.dtd"],
+        ["xmlns:content", "http://purl.org/rss/1.0/modules/content/"],
+        ["xmlns:podcast", "https://podcastindex.org/namespace/1.0"],
+      ],
+    ),
+  );
+}
+
+function assembleEpisode(options: { series: Series; episode: Episode }) {
+  const { series, episode } = options;
+
+  const description = episode.subtitle || "";
+
+  return tag("item", [
+    tag("title", episode.title),
+    tag("link", episode.shareLink),
+    tag("description", description),
+    tag("itunes:summary", description),
+    tag("guid", episode.id, [["isPermaLink", "false"]]),
+    tag("pubDate", new Date(episode.date).toUTCString()),
+    tag("itunes:duration", episode.durationInSeconds.toString()),
+    tag("podcast:chapters", "", [
+      [
+        "url",
+        `${getHostName()}/api/feeds/${series.id}/${episode.id}/chapters`,
+      ],
+      ["type", "application/json+chapters"],
+    ]),
+    tag("enclosure", "", [
+      ["url", episode.url],
+      ["length", episode.durationInSeconds.toString()],
+      ["type", "audio/mpeg3"],
+    ]),
+  ]);
+}
+
+export const rss = {
+  assembleFeed,
+};

lib/storage.test.ts

@@ -0,0 +1,21 @@
+import { assertEquals } from "$std/assert/mod.ts";
+import { storage } from "./storage.ts";
+import { testUtils } from "./test-utils.ts";
+
+Deno.test("can store a series", async () => {
+  const series = testUtils.generateSeries();
+  await storage.write(series);
+
+  const readSeries = await storage.read(series);
+
+  assertEquals(readSeries, series);
+});
+
+Deno.test("can retrieve a series", async () => {
+  const series = testUtils.generateSeries();
+  await storage.write(series);
+
+  const readSeries = await storage.read(series);
+
+  assertEquals(readSeries, series);
+});

lib/storage.ts

@@ -0,0 +1,43 @@
+export type Episode = {
+  id: string;
+  title: string;
+  subtitle: string | null;
+  url: string;
+  shareLink: string;
+  date: Date;
+  durationInSeconds: number;
+};
+
+export type Series = {
+  id: string;
+  title: string;
+  subtitle: string | null;
+  link: string;
+  imageUrl: string;
+  lastFetchedAt: Date;
+  episodes: Episode[];
+};
+
+const kv = await Deno.openKv();
+
+function seriesKey(series: { id: string }) {
+  return ["series", series.id];
+}
+
+async function read(options: { id: string }): Promise<Series | null> {
+  const { id } = options;
+  const key = seriesKey({ id });
+  const read = await kv.get<Series>(key);
+  return read.value;
+}
+
+async function write(series: Series): Promise<boolean> {
+  const key = seriesKey(series);
+  const stored = await kv.set(key, series);
+  return stored.ok;
+}
+
+export const storage = {
+  read,
+  write,
+};

lib/test-utils.ts

@@ -0,0 +1,31 @@
+import { fakerNB_NO as faker } from "npm:@faker-js/faker";
+import { Series } from "./storage.ts";
+
+function generateSeries(overrides: Partial<Series> = {}): Series {
+  return {
+    id: faker.word.words(1),
+    title: faker.word.words(3),
+    subtitle: faker.word.words(3),
+    link: faker.internet.url(),
+    imageUrl: faker.image.url(),
+    lastFetchedAt: faker.date.recent(),
+    episodes: new Array(faker.number.int({ min: 0, max: 100 }))
+      .fill(null).map(() => ({
+        title: faker.word.words(3),
+        subtitle: faker.word.words(3),
+        link: faker.internet.url(),
+        description: faker.lorem.paragraphs(2),
+        id: faker.string.uuid(),
+        date: faker.date.recent(),
+        durationInSeconds: faker.number.int({
+          min: 0,
+          max: 3600,
+        }),
+        ...overrides,
+      })),
+  };
+}
+
+export const testUtils = {
+  generateSeries,
+};

routes/api/feeds/[seriesId].ts

@@ -1,112 +1,24 @@
-import { nrkRadio, OriginalEpisode } from "../../../lib/nrk.ts";
 import { FreshContext, STATUS_CODE } from "$fresh/server.ts";
-import { declaration, serialize, tag } from "https://raw.githubusercontent.com/olaven/serialize-xml/v0.4.0/mod.ts";
+import { caching } from "../../../lib/caching.ts";
-import { getHostName } from "../../../utils.ts";
+import { rss } from "../../../lib/rss.ts";
-import { SeriesData } from "../../../lib/nrk.ts";
-
-function toItemTag(seriesId: string, episode: OriginalEpisode) {
-  const description = episode.titles.subtitle || "";
-  return tag("item", [
-    tag("title", episode.titles.title),
-    tag("link", episode._links.share?.href),
-    tag("description", description),
-    tag("itunes:summary", description),
-    tag("guid", episode.id, [["isPermaLink", "false"]]),
-    tag("pubDate", new Date(episode.date).toUTCString()),
-    tag("itunes:duration", episode.durationInSeconds.toString()),
-    tag("podcast:chapters", "", [
-      [
-        "url",
-        `${getHostName()}/api/feeds/${seriesId}/${episode.episodeId}/chapters`,
-      ],
-      ["type", "application/json+chapters"],
-    ]),
-    tag("enclosure", "", [
-      ["url", episode.url],
-      ["length", episode.durationInSeconds.toString()],
-      ["type", "audio/mpeg3"],
-    ]),
-  ]);
-}
-
-function buildFeed(series: SeriesData) {
-  const imageUrl = series.squareImage?.at(-1)?.url ?? "";
-  const linkValue = `https://radio.nrk.no/podkast/${series.id}`;
-
-  // Quickly adapted from https://raw.githubusercontent.com/olaven/paperpod/1cde9abd3174b26e126aa74fc5a3b63fd078c0fd/packages/converter/src/rss.ts
-  return serialize(
-    declaration([
-      ["version", "1.0"],
-      ["encoding", "UTF-8"],
-    ]),
-    tag(
-      "rss",
-      [
-        tag("channel", [
-          tag("title", series.titles.title),
-          tag("link", linkValue),
-          tag("itunes:author", "NRK"),
-          /* serie.category.id does not overlap with Apple's supported categories..
-                       These podcast feeds are not going to be indexed in itunes anyways, so
-                       a static, valid category is fine. The point is simply to pass third party
-                       podcast feed validation.
-                       */
-          tag("itunes:category", "", [["text", "Government"]]),
-          tag("itunes:owner", [
-            tag("itunes:name", "NRK"),
-            tag("itunes:email", "nrkpodcast@nrk.no"),
-          ]),
-          tag(
-            "description",
-            series.titles.subtitle || "",
-          ),
-          tag("ttl", "60"), //60 minutes
-          ...(imageUrl
-            ? [
-              tag("itunes:image", "", [
-                ["href", imageUrl],
-              ]),
-              tag("image", [
-                tag("url", imageUrl),
-                tag("title", series.titles.title),
-                tag("link", linkValue),
-              ]),
-            ]
-            : []),
-          ...series.episodes.map((episode) => toItemTag(series.id, episode)),
-        ]),
-      ],
-      [
-        ["version", "2.0"],
-        ["xmlns:itunes", "http://www.itunes.com/dtds/podcast-1.0.dtd"],
-        ["xmlns:content", "http://purl.org/rss/1.0/modules/content/"],
-        ["xmlns:podcast", "https://podcastindex.org/namespace/1.0"],
-      ],
-    ),
-  );
-}
 
 export const handler = async (_req: Request, ctx: FreshContext): Promise<Response> => {
   const seriesId = ctx.params.seriesId;
-  const series = await nrkRadio.getSerieData(seriesId);
-  if (!series) {
-    return new Response(`Couldn't find series with seriesId: ${seriesId}`, { status: STATUS_CODE.NotFound });
-  }
-  try {
-    const feedContent = buildFeed(series);
 
-    return new Response(feedContent, {
+  const series = await caching.getSeries({ id: seriesId });
+  if (!series) {
+    return new Response(JSON.stringify({ message: "Series not found" }), {
+      status: STATUS_CODE.NotFound,
       headers: {
         "Content-Type": "application/xml",
       },
     });
-  } catch (error) {
+  }
-    console.error(error);
+  const feed = rss.assembleFeed(series);
-    return new Response(JSON.stringify({ message: "Could not generate feed" }), {
+
-      status: STATUS_CODE.InternalServerError,
+  return new Response(feed, {
     headers: {
-        "Content-Type": "application/json",
+      "Content-Type": "application/xml",
     },
   });
-  }
 };

routes/api/feeds/[seriesId]/[episodeId]/chapters.ts

@@ -1,13 +1,13 @@
 import { FreshContext, STATUS_CODE } from "$fresh/server.ts";
 import { parse, toSeconds } from "https://esm.sh/iso8601-duration@2.1.1";
-import { nrkRadio, PodcastEpisode } from "../../../../../lib/nrk.ts";
+import { NrkPodcastEpisode, nrkRadio } from "../../../../../lib/nrk/nrk.ts";
 
 type Chapter = {
   title: string | undefined;
   startTime: number | undefined;
 };
 
-function toChapters(episode: PodcastEpisode): Chapter[] | null {
+function toChapters(episode: NrkPodcastEpisode): Chapter[] | null {
   if (!episode.indexPoints) {
     return null;
   }

routes/index.tsx

@@ -4,13 +4,13 @@ import Footer from "../components/Footer.tsx";
 import Header from "../components/Header.tsx";
 import Search from "../components/Search.tsx";
 import SeriesCard from "../components/SeriesCard.tsx";
-import { nrkRadio, SearchResultList } from "../lib/nrk.ts";
 import { CSS, render } from "$gfm";
+import { nrkRadio, NrkSearchResultList } from "../lib/nrk/nrk.ts";
 
 type Props = {
   query: string | null;
   rawMarkdown: string;
-  result?: SearchResultList | null;
+  result?: NrkSearchResultList | null;
 };
 
 export const handler: Handlers<Props> = {