diff --git a/components/SeriesCard.tsx b/components/SeriesCard.tsx index 50ed1b9..479c6ee 100644 --- a/components/SeriesCard.tsx +++ b/components/SeriesCard.tsx @@ -1,5 +1,5 @@ -import { SearchResult } from "../lib/nrk.ts"; import CopyButton from "../islands/CopyButton.tsx"; +import { SearchResult } from "../lib/nrk/nrk.ts"; export default function SeriesCard(props: { serie: SearchResult; origin: string }) { const feedUrl = new URL(`/api/feeds/${props.serie.seriesId}`, props.origin); diff --git a/deno.json b/deno.json index 865a2af..0b9fa23 100644 --- a/deno.json +++ b/deno.json @@ -8,6 +8,9 @@ "check": "deno fmt --check && deno lint && deno check **/*.ts && deno check **/*.tsx", "types:nrk": "deno run -A generate-types.ts && deno fmt ./lib" }, + "unstable": [ + "kv" + ], "imports": { "$fresh/": "https://deno.land/x/fresh@1.6.8/", "preact": "https://esm.sh/preact@10.19.6", @@ -17,6 +20,7 @@ "@preact/signals-core": "https://esm.sh/*@preact/signals-core@1.5.1", "$gfm": "https://deno.land/x/gfm@0.6.0/mod.ts", "openapi-typescript": "npm:openapi-typescript@6.7.5", + "serialize-xml": "https://raw.githubusercontent.com/olaven/serialize-xml/v0.4.0/mod.ts", "$std/": "https://deno.land/std@0.221.0/" }, "compilerOptions": { diff --git a/lib/caching.ts b/lib/caching.ts new file mode 100644 index 0000000..f54d96f --- /dev/null +++ b/lib/caching.ts @@ -0,0 +1,90 @@ +import { nrkRadio } from "./nrk/nrk.ts"; +import { Series, storage } from "./storage.ts"; +import * as datetime from "$std/datetime/mod.ts"; +import { Episode } from "./storage.ts"; + +const SYNC_INTERVAL_HOURS = 1; + +async function initialFetch(options: { id: string }): Promise { + const series = await nrkRadio.getSeries(options.id); + if (!series) { + return null; + } + const stored = storage.write(series); + if (!stored) { + console.error(`Failed to store series ${options.id}`); + return null; + } + + return series; +} + +type UpdatedSeries = { + lastFetch: Date; + episodes: Episode[]; +} & Series; +async function updateFetch(existingSeries: Series): Promise { + const series = await nrkRadio.getSeries(existingSeries.id); + if (!series) { + return null; + } + const newEpisodes = series.episodes.filter((episode) => { + return !existingSeries.episodes.find((serieEpisode) => serieEpisode.id === episode.id); + }); + + const updated = { + ...existingSeries, + lastFetch: new Date(), + /** + * Since we don't control the API, + * we should not make assumptions about the order, + * but rather sort the episode to what we want. + */ + episodes: [...newEpisodes, ...existingSeries.episodes] + .sort((a, b) => a.date.getTime() > b.date.getTime() ? -1 : 1), + }; + + const updateSuccessful = await storage.write(updated); + if (!updateSuccessful) { + throw new Error(`Failed to update series ${existingSeries.id}`); + } + + return updated; +} + +async function getSeries(options: { id: string }): Promise { + const seriesFromStorage = await storage.read(options); + + /** + * We don't have the feed in storage, + * and we need to fetch it for the first time. + */ + if (seriesFromStorage === null) { + // TODO: schedule a job to fetch the entire backlog https://github.com/olaven/NRSS/issues/24 + return await initialFetch(options); + } + + const timeSinceLastFetch = + datetime.difference(seriesFromStorage.lastFetchedAt, new Date(), { units: ["hours"] }).hours; + + // we have the feed in storage and it's not too old + if ( + seriesFromStorage !== null && + timeSinceLastFetch !== null && + timeSinceLastFetch !== undefined && + timeSinceLastFetch <= SYNC_INTERVAL_HOURS + ) { + return seriesFromStorage; + } + + /** + * We have the feed in storage, but it's too old + * and needs to be refreshed. + */ + const updatedSeries = await updateFetch(seriesFromStorage); + return updatedSeries; +} + +export const caching = { + getSeries, +}; diff --git a/lib/nrk-catalog.ts b/lib/nrk/nrk-catalog.ts similarity index 100% rename from lib/nrk-catalog.ts rename to lib/nrk/nrk-catalog.ts diff --git a/lib/nrk-playback.ts b/lib/nrk/nrk-playback.ts similarity index 100% rename from lib/nrk-playback.ts rename to lib/nrk/nrk-playback.ts diff --git a/lib/nrk-search.ts b/lib/nrk/nrk-search.ts similarity index 100% rename from lib/nrk-search.ts rename to lib/nrk/nrk-search.ts diff --git a/lib/nrk.ts b/lib/nrk/nrk.ts similarity index 69% rename from lib/nrk.ts rename to lib/nrk/nrk.ts index a2f0520..5e97f3e 100644 --- a/lib/nrk.ts +++ b/lib/nrk/nrk.ts @@ -2,6 +2,7 @@ import { get, STATUS_CODE } from "https://deno.land/x/kall@v2.0.0/mod.ts"; import { components as searchComponents } from "./nrk-search.ts"; import { components as catalogComponents } from "./nrk-catalog.ts"; import { external as playbackComponents } from "./nrk-playback.ts"; +import { Series } from "../storage.ts"; type ArrayElement = A extends readonly (infer T)[] ? T : never; type PodcastEpisodes = catalogComponents["schemas"]["EpisodesHalResource"]; @@ -10,17 +11,42 @@ type PodcastEpisodesSingle = catalogComponents["schemas"]["EpisodeHalResource"]; type RadioSeriesEpisode = catalogComponents["schemas"]["EpisodesHalResource"]; type RadioSeries = catalogComponents["schemas"]["SeriesHalResource"]; -export type Serie = catalogComponents["schemas"]["SeriesViewModel"]; -export type OriginalEpisode = PodcastEpisodesSingle & { url: string }; -export type PodcastEpisode = catalogComponents["schemas"]["PodcastEpisodeHalResource"]; -export type SearchResultList = searchComponents["schemas"]["seriesResult"]["results"]; -export type SearchResult = ArrayElement & { +export type NrkSerie = catalogComponents["schemas"]["SeriesViewModel"]; +export type NrkOriginalEpisode = PodcastEpisodesSingle & { url: string }; +export type NrkPodcastEpisode = catalogComponents["schemas"]["PodcastEpisodeHalResource"]; +export type NrkSearchResultList = searchComponents["schemas"]["seriesResult"]["results"]; +export type SearchResult = ArrayElement & { description?: string; }; +export type SeriesData = { episodes: NrkOriginalEpisode[] } & (RadioSeries["series"] | Podcast["series"]); + +function parseSeries(nrkSeries: SeriesData): Series { + const imageUrl = nrkSeries.squareImage?.at(-1)?.url ?? ""; + + return { + id: nrkSeries.id, + title: nrkSeries.titles.title, + subtitle: nrkSeries.titles.subtitle ?? null, + link: `https://radio.nrk.no/podkast/${nrkSeries.id}`, + imageUrl: imageUrl, + lastFetchedAt: new Date(), + episodes: nrkSeries.episodes.map((episode) => { + return { + id: episode.id, + title: episode.titles.title, + subtitle: episode.titles.subtitle ?? null, + url: episode.url, + shareLink: episode._links.share?.href ?? "", + date: new Date(episode.date), + durationInSeconds: episode.durationInSeconds, + }; + }), + }; +} const nrkAPI = `https://psapi.nrk.no`; -async function search(query: string): Promise { +async function search(query: string): Promise { if (query === "") { console.error("Empty search query."); return null; @@ -36,8 +62,7 @@ async function search(query: string): Promise { return null; } -export type SeriesData = { episodes: OriginalEpisode[] } & (RadioSeries["series"] | Podcast["series"]); -async function getSerieData(seriesId: string): Promise { +async function getSeries(seriesId: string): Promise { let [ { status: episodeStatus, body: episodeResponse }, { status: seriesStatus, body: serieResponse }, @@ -71,10 +96,12 @@ async function getSerieData(seriesId: string): Promise { const episodes = await Promise.all( episodeResponse._embedded.episodes.map((episode) => getEpisodeWithDownloadLink(episode, serieResponse.type)), ); - return { + const seriesData = { ...serieResponse.series, episodes, }; + const parsedSeries = parseSeries(seriesData); + return parsedSeries; } console.error( `Error getting episodes for ${seriesId}: EpisodeStatus: ${episodeStatus}. SerieStatus: ${seriesStatus}`, @@ -82,9 +109,9 @@ async function getSerieData(seriesId: string): Promise { return null; } -async function getEpisode(seriesId: string, episodeId: string): Promise { +async function getEpisode(seriesId: string, episodeId: string): Promise { const url = `${nrkAPI}/radio/catalog/podcast/${seriesId}/episodes/${episodeId}`; - const { status, body: episode } = await get(url); + const { status, body: episode } = await get(url); if (status === STATUS_CODE.OK && episode) { return episode; } @@ -97,7 +124,7 @@ type Manifest = playbackComponents["schemas/playback-channel.json"]["components" async function getEpisodeWithDownloadLink( episode: PodcastEpisodesSingle, type: catalogComponents["schemas"]["Type"], -): Promise { +): Promise { // getting stream link let { status: playbackStatus, body: playbackResponse } = await get( `${nrkAPI}/playback/manifest/podcast/${episode.episodeId}`, @@ -120,10 +147,7 @@ async function getEpisodeWithDownloadLink( export const nrkRadio = { search, - getSerieData, + getSeries, getEpisode, -}; - -export const forTestingOnly = { - getEpisodeWithDownloadLink, + parseSeries, }; diff --git a/lib/nrk_test.ts b/lib/nrk/nrk_test.ts similarity index 74% rename from lib/nrk_test.ts rename to lib/nrk/nrk_test.ts index 26297bc..98978c9 100644 --- a/lib/nrk_test.ts +++ b/lib/nrk/nrk_test.ts @@ -2,7 +2,6 @@ import { assertEquals } from "$std/assert/assert_equals.ts"; import { nrkRadio } from "./nrk.ts"; import { assertGreaterOrEqual } from "$std/assert/assert_greater_or_equal.ts"; import { assertExists } from "https://deno.land/std@0.216.0/assert/assert_exists.ts"; -import { forTestingOnly } from "./nrk.ts"; Deno.test("Verify search query `trygd` returns one result: 'Trygdekontoret'", async () => { const result = await nrkRadio.search("trygd"); @@ -18,14 +17,14 @@ Deno.test("Verify empty search query yields `null`", async () => { }); Deno.test("Verify getting series data for 'trygdekontoret' works", async () => { - const result = await nrkRadio.getSerieData("trygdekontoret"); + const result = await nrkRadio.getSeries("trygdekontoret"); assertExists(result); assertGreaterOrEqual(result.episodes.length, 1); }); Deno.test("Verify getting series data for 'trygd' does not works", async () => { - const result = await nrkRadio.getSerieData("trygd"); + const result = await nrkRadio.getSeries("trygd"); assertEquals(result, null); }); @@ -41,11 +40,3 @@ Deno.test("Verify getting episodeId 'null' for 'trygdekontoret' yields `null`", assertEquals(result, null); }); - -Deno.test("Verify getting episode, 'l_0bc5e55a-46b5-48a5-85e5-5a46b5d8a562' with download link works`", async () => { - const series = await nrkRadio.getSerieData("trygdekontoret"); - assertExists(series); - - const result = await forTestingOnly.getEpisodeWithDownloadLink(series.episodes[0], "podcast"); - assertExists(result); -}); diff --git a/lib/rss.test.ts b/lib/rss.test.ts new file mode 100644 index 0000000..635c609 --- /dev/null +++ b/lib/rss.test.ts @@ -0,0 +1,19 @@ +import { assertEquals, assertExists } from "$std/assert/mod.ts"; +import { testUtils } from "./test-utils.ts"; +import { rss } from "./rss.ts"; + +Deno.test("generated rss contains the series title", () => { + const series = testUtils.generateSeries(); + const feed = rss.assembleFeed(series); + assertExists(feed); + assertEquals(feed.includes(series.title), true); +}); + +Deno.test("generated rss contains all episode titles", () => { + const series = testUtils.generateSeries(); + const feed = rss.assembleFeed(series); + assertExists(feed); + series.episodes.forEach((episode) => { + assertEquals(feed.includes(episode.title), true); + }); +}); diff --git a/lib/rss.ts b/lib/rss.ts new file mode 100644 index 0000000..7e2a025 --- /dev/null +++ b/lib/rss.ts @@ -0,0 +1,90 @@ +import { declaration, serialize, tag } from "serialize-xml"; +import { getHostName } from "../utils.ts"; +import { Episode, Series } from "./storage.ts"; + +function assembleFeed(series: Series) { + // Originally adapted from https://raw.githubusercontent.com/olaven/paperpod/1cde9abd3174b26e126aa74fc5a3b63fd078c0fd/packages/converter/src/rss.ts + return serialize( + declaration([ + ["version", "1.0"], + ["encoding", "UTF-8"], + ]), + tag( + "rss", + [ + tag("channel", [ + tag("title", series.title), + tag("link", series.link), + tag("itunes:author", "NRK"), + /** + * serie.category.id does not overlap with Apple's supported categories.. + * These podcast feeds are not going to be indexed in itunes anyways, so + * a static, valid category is fine. The point is simply to pass third party + * podcast feed validation. + */ + tag("itunes:category", "", [["text", "Government"]]), + tag("itunes:owner", [ + tag("itunes:name", "NRK"), + tag("itunes:email", "nrkpodcast@nrk.no"), + ]), + tag( + "description", + series.subtitle || "", + ), + tag("ttl", "60"), //60 minutes + ...(series.imageUrl + ? [ + tag("itunes:image", "", [ + ["href", series.imageUrl], + ]), + tag("image", [ + tag("url", series.imageUrl), + tag("title", series.title), + tag("link", series.link), + ]), + ] + : []), + ...series.episodes.map((episode) => assembleEpisode({ series, episode })), + ]), + ], + [ + ["version", "2.0"], + ["xmlns:itunes", "http://www.itunes.com/dtds/podcast-1.0.dtd"], + ["xmlns:content", "http://purl.org/rss/1.0/modules/content/"], + ["xmlns:podcast", "https://podcastindex.org/namespace/1.0"], + ], + ), + ); +} + +function assembleEpisode(options: { series: Series; episode: Episode }) { + const { series, episode } = options; + + const description = episode.subtitle || ""; + + return tag("item", [ + tag("title", episode.title), + tag("link", episode.shareLink), + tag("description", description), + tag("itunes:summary", description), + tag("guid", episode.id, [["isPermaLink", "false"]]), + tag("pubDate", new Date(episode.date).toUTCString()), + tag("itunes:duration", episode.durationInSeconds.toString()), + tag("podcast:chapters", "", [ + [ + "url", + `${getHostName()}/api/feeds/${series.id}/${episode.id}/chapters`, + ], + ["type", "application/json+chapters"], + ]), + tag("enclosure", "", [ + ["url", episode.url], + ["length", episode.durationInSeconds.toString()], + ["type", "audio/mpeg3"], + ]), + ]); +} + +export const rss = { + assembleFeed, +}; diff --git a/lib/storage.test.ts b/lib/storage.test.ts new file mode 100644 index 0000000..602374f --- /dev/null +++ b/lib/storage.test.ts @@ -0,0 +1,21 @@ +import { assertEquals } from "$std/assert/mod.ts"; +import { storage } from "./storage.ts"; +import { testUtils } from "./test-utils.ts"; + +Deno.test("can store a series", async () => { + const series = testUtils.generateSeries(); + await storage.write(series); + + const readSeries = await storage.read(series); + + assertEquals(readSeries, series); +}); + +Deno.test("can retrieve a series", async () => { + const series = testUtils.generateSeries(); + await storage.write(series); + + const readSeries = await storage.read(series); + + assertEquals(readSeries, series); +}); diff --git a/lib/storage.ts b/lib/storage.ts new file mode 100644 index 0000000..6b8e2db --- /dev/null +++ b/lib/storage.ts @@ -0,0 +1,43 @@ +export type Episode = { + id: string; + title: string; + subtitle: string | null; + url: string; + shareLink: string; + date: Date; + durationInSeconds: number; +}; + +export type Series = { + id: string; + title: string; + subtitle: string | null; + link: string; + imageUrl: string; + lastFetchedAt: Date; + episodes: Episode[]; +}; + +const kv = await Deno.openKv(); + +function seriesKey(series: { id: string }) { + return ["series", series.id]; +} + +async function read(options: { id: string }): Promise { + const { id } = options; + const key = seriesKey({ id }); + const read = await kv.get(key); + return read.value; +} + +async function write(series: Series): Promise { + const key = seriesKey(series); + const stored = await kv.set(key, series); + return stored.ok; +} + +export const storage = { + read, + write, +}; diff --git a/lib/test-utils.ts b/lib/test-utils.ts new file mode 100644 index 0000000..8d0ea08 --- /dev/null +++ b/lib/test-utils.ts @@ -0,0 +1,31 @@ +import { fakerNB_NO as faker } from "npm:@faker-js/faker"; +import { Series } from "./storage.ts"; + +function generateSeries(overrides: Partial = {}): Series { + return { + id: faker.word.words(1), + title: faker.word.words(3), + subtitle: faker.word.words(3), + link: faker.internet.url(), + imageUrl: faker.image.url(), + lastFetchedAt: faker.date.recent(), + episodes: new Array(faker.number.int({ min: 0, max: 100 })) + .fill(null).map(() => ({ + title: faker.word.words(3), + subtitle: faker.word.words(3), + link: faker.internet.url(), + description: faker.lorem.paragraphs(2), + id: faker.string.uuid(), + date: faker.date.recent(), + durationInSeconds: faker.number.int({ + min: 0, + max: 3600, + }), + ...overrides, + })), + }; +} + +export const testUtils = { + generateSeries, +}; diff --git a/routes/api/feeds/[seriesId].ts b/routes/api/feeds/[seriesId].ts index 4e78e02..e346a03 100644 --- a/routes/api/feeds/[seriesId].ts +++ b/routes/api/feeds/[seriesId].ts @@ -1,112 +1,24 @@ -import { nrkRadio, OriginalEpisode } from "../../../lib/nrk.ts"; import { FreshContext, STATUS_CODE } from "$fresh/server.ts"; -import { declaration, serialize, tag } from "https://raw.githubusercontent.com/olaven/serialize-xml/v0.4.0/mod.ts"; -import { getHostName } from "../../../utils.ts"; -import { SeriesData } from "../../../lib/nrk.ts"; - -function toItemTag(seriesId: string, episode: OriginalEpisode) { - const description = episode.titles.subtitle || ""; - return tag("item", [ - tag("title", episode.titles.title), - tag("link", episode._links.share?.href), - tag("description", description), - tag("itunes:summary", description), - tag("guid", episode.id, [["isPermaLink", "false"]]), - tag("pubDate", new Date(episode.date).toUTCString()), - tag("itunes:duration", episode.durationInSeconds.toString()), - tag("podcast:chapters", "", [ - [ - "url", - `${getHostName()}/api/feeds/${seriesId}/${episode.episodeId}/chapters`, - ], - ["type", "application/json+chapters"], - ]), - tag("enclosure", "", [ - ["url", episode.url], - ["length", episode.durationInSeconds.toString()], - ["type", "audio/mpeg3"], - ]), - ]); -} - -function buildFeed(series: SeriesData) { - const imageUrl = series.squareImage?.at(-1)?.url ?? ""; - const linkValue = `https://radio.nrk.no/podkast/${series.id}`; - - // Quickly adapted from https://raw.githubusercontent.com/olaven/paperpod/1cde9abd3174b26e126aa74fc5a3b63fd078c0fd/packages/converter/src/rss.ts - return serialize( - declaration([ - ["version", "1.0"], - ["encoding", "UTF-8"], - ]), - tag( - "rss", - [ - tag("channel", [ - tag("title", series.titles.title), - tag("link", linkValue), - tag("itunes:author", "NRK"), - /* serie.category.id does not overlap with Apple's supported categories.. - These podcast feeds are not going to be indexed in itunes anyways, so - a static, valid category is fine. The point is simply to pass third party - podcast feed validation. - */ - tag("itunes:category", "", [["text", "Government"]]), - tag("itunes:owner", [ - tag("itunes:name", "NRK"), - tag("itunes:email", "nrkpodcast@nrk.no"), - ]), - tag( - "description", - series.titles.subtitle || "", - ), - tag("ttl", "60"), //60 minutes - ...(imageUrl - ? [ - tag("itunes:image", "", [ - ["href", imageUrl], - ]), - tag("image", [ - tag("url", imageUrl), - tag("title", series.titles.title), - tag("link", linkValue), - ]), - ] - : []), - ...series.episodes.map((episode) => toItemTag(series.id, episode)), - ]), - ], - [ - ["version", "2.0"], - ["xmlns:itunes", "http://www.itunes.com/dtds/podcast-1.0.dtd"], - ["xmlns:content", "http://purl.org/rss/1.0/modules/content/"], - ["xmlns:podcast", "https://podcastindex.org/namespace/1.0"], - ], - ), - ); -} +import { caching } from "../../../lib/caching.ts"; +import { rss } from "../../../lib/rss.ts"; export const handler = async (_req: Request, ctx: FreshContext): Promise => { const seriesId = ctx.params.seriesId; - const series = await nrkRadio.getSerieData(seriesId); - if (!series) { - return new Response(`Couldn't find series with seriesId: ${seriesId}`, { status: STATUS_CODE.NotFound }); - } - try { - const feedContent = buildFeed(series); - return new Response(feedContent, { + const series = await caching.getSeries({ id: seriesId }); + if (!series) { + return new Response(JSON.stringify({ message: "Series not found" }), { + status: STATUS_CODE.NotFound, headers: { "Content-Type": "application/xml", }, }); - } catch (error) { - console.error(error); - return new Response(JSON.stringify({ message: "Could not generate feed" }), { - status: STATUS_CODE.InternalServerError, - headers: { - "Content-Type": "application/json", - }, - }); } + const feed = rss.assembleFeed(series); + + return new Response(feed, { + headers: { + "Content-Type": "application/xml", + }, + }); }; diff --git a/routes/api/feeds/[seriesId]/[episodeId]/chapters.ts b/routes/api/feeds/[seriesId]/[episodeId]/chapters.ts index d6d0236..d515d04 100644 --- a/routes/api/feeds/[seriesId]/[episodeId]/chapters.ts +++ b/routes/api/feeds/[seriesId]/[episodeId]/chapters.ts @@ -1,13 +1,13 @@ import { FreshContext, STATUS_CODE } from "$fresh/server.ts"; import { parse, toSeconds } from "https://esm.sh/iso8601-duration@2.1.1"; -import { nrkRadio, PodcastEpisode } from "../../../../../lib/nrk.ts"; +import { NrkPodcastEpisode, nrkRadio } from "../../../../../lib/nrk/nrk.ts"; type Chapter = { title: string | undefined; startTime: number | undefined; }; -function toChapters(episode: PodcastEpisode): Chapter[] | null { +function toChapters(episode: NrkPodcastEpisode): Chapter[] | null { if (!episode.indexPoints) { return null; } diff --git a/routes/index.tsx b/routes/index.tsx index dc55011..d10efbb 100644 --- a/routes/index.tsx +++ b/routes/index.tsx @@ -4,13 +4,13 @@ import Footer from "../components/Footer.tsx"; import Header from "../components/Header.tsx"; import Search from "../components/Search.tsx"; import SeriesCard from "../components/SeriesCard.tsx"; -import { nrkRadio, SearchResultList } from "../lib/nrk.ts"; import { CSS, render } from "$gfm"; +import { nrkRadio, NrkSearchResultList } from "../lib/nrk/nrk.ts"; type Props = { query: string | null; rawMarkdown: string; - result?: SearchResultList | null; + result?: NrkSearchResultList | null; }; export const handler: Handlers = {