From b632ea4f95d42d1e468b603c7e3ee07c97afc2e8 Mon Sep 17 00:00:00 2001 From: olaven Date: Sat, 6 Apr 2024 23:39:16 +0200 Subject: [PATCH] WIP: caching --- deno.json | 10 ++- lib/caching.ts | 28 ++++++++ lib/{ => nrk}/nrk-catalog.ts | 0 lib/{ => nrk}/nrk-playback.ts | 0 lib/{ => nrk}/nrk-search.ts | 0 lib/{ => nrk}/nrk.ts | 0 lib/parse.ts | 31 +++++++++ lib/rss.test.ts | 23 +++++++ lib/rss.ts | 93 ++++++++++++++++++++++++++ lib/storage.test.ts | 23 +++++++ lib/storage.ts | 42 ++++++++++++ lib/test-utils.ts | 30 +++++++++ routes/api/feeds/[seriesId].ts | 118 ++++++--------------------------- 13 files changed, 298 insertions(+), 100 deletions(-) create mode 100644 lib/caching.ts rename lib/{ => nrk}/nrk-catalog.ts (100%) rename lib/{ => nrk}/nrk-playback.ts (100%) rename lib/{ => nrk}/nrk-search.ts (100%) rename lib/{ => nrk}/nrk.ts (100%) create mode 100644 lib/parse.ts create mode 100644 lib/rss.test.ts create mode 100644 lib/rss.ts create mode 100644 lib/storage.test.ts create mode 100644 lib/storage.ts create mode 100644 lib/test-utils.ts diff --git a/deno.json b/deno.json index e8e12b5..9b45de3 100644 --- a/deno.json +++ b/deno.json @@ -8,6 +8,9 @@ "check": "deno fmt --check && deno lint && deno check **/*.ts && deno check **/*.tsx", "types:nrk": "deno run -A generate-types.ts && deno fmt ./lib" }, + "unstable": [ + "kv" + ], "imports": { "$fresh/": "https://deno.land/x/fresh@1.6.8/", "preact": "https://esm.sh/preact@10.19.6", @@ -16,7 +19,10 @@ "@preact/signals": "https://esm.sh/*@preact/signals@1.2.2", "@preact/signals-core": "https://esm.sh/*@preact/signals-core@1.5.1", "$gfm": "https://deno.land/x/gfm@0.6.0/mod.ts", - "openapi-typescript": "npm:openapi-typescript@6.7.5" + "openapi-typescript": "npm:openapi-typescript@6.7.5", + "datetime": "https://deno.land/std@0.221.0/datetime/mod.ts", + "asserts": "https://deno.land/std@0.221.0/assert/mod.ts", + "serialize-xml": "https://raw.githubusercontent.com/olaven/serialize-xml/v0.4.0/mod.ts" }, "compilerOptions": { "jsx": "react-jsx", @@ -39,4 +45,4 @@ ] } } -} +} \ No newline at end of file diff --git a/lib/caching.ts b/lib/caching.ts new file mode 100644 index 0000000..68396ef --- /dev/null +++ b/lib/caching.ts @@ -0,0 +1,28 @@ +import * as datetime from "datetime" +import { Series } from "./storage.ts" + +const UPDATE_INTERVAL_HOURS = 1; +function getAction(options: { + existingSeries: Series | null, +}) { + const { existingSeries } = options; + if (existingSeries === null) { + return "INITIAL_FETCH"; + } + + const timeSinceLastFetch = datetime.difference( + new Date(), + existingSeries.lastFetched, + { + units: ["hours"], + } + ) + + if (timeSinceLastFetch.hours && + timeSinceLastFetch.hours > UPDATE_INTERVAL_HOURS) { + return "UPDATE"; + } + + return "RETURN_AS_IS"; +} + diff --git a/lib/nrk-catalog.ts b/lib/nrk/nrk-catalog.ts similarity index 100% rename from lib/nrk-catalog.ts rename to lib/nrk/nrk-catalog.ts diff --git a/lib/nrk-playback.ts b/lib/nrk/nrk-playback.ts similarity index 100% rename from lib/nrk-playback.ts rename to lib/nrk/nrk-playback.ts diff --git a/lib/nrk-search.ts b/lib/nrk/nrk-search.ts similarity index 100% rename from lib/nrk-search.ts rename to lib/nrk/nrk-search.ts diff --git a/lib/nrk.ts b/lib/nrk/nrk.ts similarity index 100% rename from lib/nrk.ts rename to lib/nrk/nrk.ts diff --git a/lib/parse.ts b/lib/parse.ts new file mode 100644 index 0000000..05f8292 --- /dev/null +++ b/lib/parse.ts @@ -0,0 +1,31 @@ +import { nrkRadio } from "./nrk/nrk.ts"; +import { Series } from "./storage.ts"; + +type NrkSerieData = Awaited> +function series(nrkSeries: NrkSerieData): Series { + + const imageUrl = nrkSeries.squareImage?.at(-1)?.url ?? ""; + + return { + id: nrkSeries.id, + title: nrkSeries.titles.title, + subtitle: nrkSeries.titles.subtitle ?? null, + link: `https://radio.nrk.no/podkast/${nrkSeries.id}`, + imageUrl: imageUrl, + lastFetched: new Date(), + episodes: nrkSeries.episodes.map(episode => { + return { + id: episode.id, + title: episode.titles.title, + subtitle: episode.titles.subtitle ?? null, + link: episode._links.share?.href ?? "", + date: new Date(episode.date), + durationInSeconds: episode.durationInSeconds, + } + }), + } +} + +export const parse = { + series, +} \ No newline at end of file diff --git a/lib/rss.test.ts b/lib/rss.test.ts new file mode 100644 index 0000000..6c92cdf --- /dev/null +++ b/lib/rss.test.ts @@ -0,0 +1,23 @@ +import { assertEquals, assertExists } from "asserts"; +import { testUtils } from "./test-utils.ts"; +import { rss } from "./rss.ts"; + + + +Deno.test("generated rss contains the series title", () => { + + const series = testUtils.generateSeries(); + const feed = rss.assembleFeed(series); + assertExists(feed) + assertEquals(feed.includes(series.title), true) +}); + +Deno.test("generated rss contains all episode titles", () => { + + const series = testUtils.generateSeries(); + const feed = rss.assembleFeed(series); + assertExists(feed) + series.episodes.forEach(episode => { + assertEquals(feed.includes(episode.title), true) + }) +}) \ No newline at end of file diff --git a/lib/rss.ts b/lib/rss.ts new file mode 100644 index 0000000..81047e5 --- /dev/null +++ b/lib/rss.ts @@ -0,0 +1,93 @@ +import { declaration, serialize, tag } from "serialize-xml"; +import { getHostName } from "../utils.ts"; +import { Episode, Series } from "./storage.ts"; + +function assembleFeed(series: Series) { + + // Originally adapted from https://raw.githubusercontent.com/olaven/paperpod/1cde9abd3174b26e126aa74fc5a3b63fd078c0fd/packages/converter/src/rss.ts + return serialize( + declaration([ + ["version", "1.0"], + ["encoding", "UTF-8"], + ]), + tag( + "rss", + [ + tag("channel", [ + tag("title", series.title), + tag("link", series.link), + tag("itunes:author", "NRK"), + /** + * serie.category.id does not overlap with Apple's supported categories.. + * These podcast feeds are not going to be indexed in itunes anyways, so + * a static, valid category is fine. The point is simply to pass third party + * podcast feed validation. + */ + tag("itunes:category", "", [["text", "Government"]]), + tag("itunes:owner", [ + tag("itunes:name", "NRK"), + tag("itunes:email", "nrkpodcast@nrk.no"), + ]), + tag( + "description", + series.subtitle || "", + ), + tag("ttl", "60"), //60 minutes + ...(series.imageUrl + ? [ + tag("itunes:image", "", [ + ["href", series.imageUrl], + ]), + tag("image", [ + tag("url", series.imageUrl), + tag("title", series.title), + tag("link", series.link), + ]), + ] + : []), + ...series.episodes.map((episode) => assembleEpisode({ series, episode })), + ]), + ], + [ + ["version", "2.0"], + ["xmlns:itunes", "http://www.itunes.com/dtds/podcast-1.0.dtd"], + ["xmlns:content", "http://purl.org/rss/1.0/modules/content/"], + ["xmlns:podcast", "https://podcastindex.org/namespace/1.0"], + ], + ), + ); +} + +function assembleEpisode(options: { series: Series, episode: Episode }) { + + const { series, episode } = options; + + const description = episode.subtitle || ""; + + return tag("item", [ + tag("title", episode.title), + tag("link", episode.link), + tag("description", description), + tag("itunes:summary", description), + tag("guid", episode.id, [["isPermaLink", "false"]]), + tag("pubDate", new Date(episode.date).toUTCString()), + tag("itunes:duration", episode.durationInSeconds.toString()), + tag("podcast:chapters", "", [ + [ + "url", + `${getHostName()}/api/feeds/${series.id}/${episode.id}/chapters`, + ], + ["type", "application/json+chapters"], + ]), + tag("enclosure", "", [ + ["url", episode.link], + ["length", episode.durationInSeconds.toString()], + ["type", "audio/mpeg3"], + ]), + ]); +} + + +export const rss = { + assembleFeed +} \ No newline at end of file diff --git a/lib/storage.test.ts b/lib/storage.test.ts new file mode 100644 index 0000000..5d62171 --- /dev/null +++ b/lib/storage.test.ts @@ -0,0 +1,23 @@ +import { assertEquals } from "asserts"; +import { storage } from "./storage.ts"; +import { testUtils } from "./test-utils.ts"; + + +Deno.test("can store a series", async () => { + + const series = testUtils.generateSeries(); + await storage.write(series); + + const readSeries = await storage.read(series); + + assertEquals(readSeries, series); +}); + +Deno.test("can retrieve a series", async () => { + const series = testUtils.generateSeries(); + await storage.write(series); + + const readSeries = await storage.read(series); + + assertEquals(readSeries, series); +}); \ No newline at end of file diff --git a/lib/storage.ts b/lib/storage.ts new file mode 100644 index 0000000..75a8b9a --- /dev/null +++ b/lib/storage.ts @@ -0,0 +1,42 @@ +export type Episode = { + id: string, + title: string, + subtitle: string | null; + link: string, + date: Date, + durationInSeconds: number, +} + +export type Series = { + id: string, + title: string, + subtitle: string | null; + link: string, + imageUrl: string, + lastFetched: Date, + episodes: Episode[], +} + +const kv = await Deno.openKv(); + +function seriesKey(series: { id: string }) { + return ["series", series.id]; +} + +async function read(options: { id: string }): Promise { + const { id } = options; + const key = seriesKey({ id }); + const read = await kv.get(key); + return read.value; +} + +async function write(series: Series): Promise { + const key = seriesKey(series); + const stored = await kv.set(key, series); + return stored.ok; +} + +export const storage = { + read, + write, +} \ No newline at end of file diff --git a/lib/test-utils.ts b/lib/test-utils.ts new file mode 100644 index 0000000..ad1c643 --- /dev/null +++ b/lib/test-utils.ts @@ -0,0 +1,30 @@ +import { fakerNB_NO as faker } from "npm:@faker-js/faker" +import { Series } from "./storage.ts"; + +function generateSeries(overrides: Partial = {}): Series { + return { + id: faker.word.words(1), + title: faker.word.words(3), + subtitle: faker.word.words(3), + link: faker.internet.url(), + imageUrl: faker.image.url(), + lastFetched: faker.date.recent(), + episodes: new Array(faker.number.int({ min: 0, max: 100 })) + .fill(null).map(() => ({ + title: faker.word.words(3), + subtitle: faker.word.words(3), + link: faker.internet.url(), + description: faker.lorem.paragraphs(2), + id: faker.string.uuid(), + date: faker.date.recent(), + durationInSeconds: faker.number.int({ + min: 0, max: 3600 + }), + ...overrides, + })) + } +} + +export const testUtils = { + generateSeries, +} \ No newline at end of file diff --git a/routes/api/feeds/[seriesId].ts b/routes/api/feeds/[seriesId].ts index 8729249..49ab837 100644 --- a/routes/api/feeds/[seriesId].ts +++ b/routes/api/feeds/[seriesId].ts @@ -1,108 +1,30 @@ -import { nrkRadio, OriginalEpisode } from "../../../lib/nrk.ts"; import { FreshContext } from "$fresh/server.ts"; -import { declaration, serialize, tag } from "https://raw.githubusercontent.com/olaven/serialize-xml/v0.4.0/mod.ts"; -import { getHostName } from "../../../utils.ts"; +import { storage } from "../../../lib/storage.ts"; + -function toItemTag(seriesId: string, episode: OriginalEpisode) { - const description = episode.titles.subtitle || ""; - return tag("item", [ - tag("title", episode.titles.title), - tag("link", episode._links.share?.href), - tag("description", description), - tag("itunes:summary", description), - tag("guid", episode.id, [["isPermaLink", "false"]]), - tag("pubDate", new Date(episode.date).toUTCString()), - tag("itunes:duration", episode.durationInSeconds.toString()), - tag("podcast:chapters", "", [ - [ - "url", - `${getHostName()}/api/feeds/${seriesId}/${episode.episodeId}/chapters`, - ], - ["type", "application/json+chapters"], - ]), - tag("enclosure", "", [ - ["url", episode.url], - ["length", episode.durationInSeconds.toString()], - ["type", "audio/mpeg3"], - ]), - ]); -} -async function buildFeed(seriesId: string) { - const serie = await nrkRadio.getSerieData(seriesId); - const imageUrl = serie.squareImage?.at(-1)?.url ?? ""; - const linkValue = `https://radio.nrk.no/podkast/${serie.id}`; - // Quickly adapted from https://raw.githubusercontent.com/olaven/paperpod/1cde9abd3174b26e126aa74fc5a3b63fd078c0fd/packages/converter/src/rss.ts - return serialize( - declaration([ - ["version", "1.0"], - ["encoding", "UTF-8"], - ]), - tag( - "rss", - [ - tag("channel", [ - tag("title", serie.titles.title), - tag("link", linkValue), - tag("itunes:author", "NRK"), - /* serie.category.id does not overlap with Apple's supported categories.. - These podcast feeds are not going to be indexed in itunes anyways, so - a static, valid category is fine. The point is simply to pass third party - podcast feed validation. - */ - tag("itunes:category", "", [["text", "Government"]]), - tag("itunes:owner", [ - tag("itunes:name", "NRK"), - tag("itunes:email", "nrkpodcast@nrk.no"), - ]), - tag( - "description", - serie.titles.subtitle || "", - ), - tag("ttl", "60"), //60 minutes - ...(imageUrl - ? [ - tag("itunes:image", "", [ - ["href", imageUrl], - ]), - tag("image", [ - tag("url", imageUrl), - tag("title", serie.titles.title), - tag("link", linkValue), - ]), - ] - : []), - ...serie.episodes.map((episode) => toItemTag(seriesId, episode)), - ]), - ], - [ - ["version", "2.0"], - ["xmlns:itunes", "http://www.itunes.com/dtds/podcast-1.0.dtd"], - ["xmlns:content", "http://purl.org/rss/1.0/modules/content/"], - ["xmlns:podcast", "https://podcastindex.org/namespace/1.0"], - ], - ), - ); -} export const handler = async (_req: Request, ctx: FreshContext): Promise => { const seriesId = ctx.params.seriesId; - try { + + const storedSeries = await storage.read({ id: seriesId }); + if (storedSeries === null) { + + + // TODO: fetch for the first time + // const convert and store + } + + if (datetime) + + + const feedContent = await buildFeed(seriesId); - return new Response(feedContent, { - headers: { - "Content-Type": "application/xml", - }, - }); - } catch (error) { - console.error(error); - return new Response(JSON.stringify({ message: "Could not generate feed" }), { - status: 500, - headers: { - "Content-Type": "application/json", - }, - }); - } + return new Response(feedContent, { + headers: { + "Content-Type": "application/xml", + }, + }) };