diff --git a/components/SeriesCard.tsx b/components/SeriesCard.tsx index 50ed1b9..479c6ee 100644 --- a/components/SeriesCard.tsx +++ b/components/SeriesCard.tsx @@ -1,5 +1,5 @@ -import { SearchResult } from "../lib/nrk.ts"; import CopyButton from "../islands/CopyButton.tsx"; +import { SearchResult } from "../lib/nrk/nrk.ts"; export default function SeriesCard(props: { serie: SearchResult; origin: string }) { const feedUrl = new URL(`/api/feeds/${props.serie.seriesId}`, props.origin); diff --git a/deno.json b/deno.json index 9b45de3..f1f68ed 100644 --- a/deno.json +++ b/deno.json @@ -45,4 +45,4 @@ ] } } -} \ No newline at end of file +} diff --git a/lib/caching.ts b/lib/caching.ts index 68396ef..41a20a7 100644 --- a/lib/caching.ts +++ b/lib/caching.ts @@ -1,28 +1,83 @@ -import * as datetime from "datetime" -import { Series } from "./storage.ts" +import { nrkRadio } from "./nrk/nrk.ts"; +import { parse } from "./parse.ts"; +import { Series, storage } from "./storage.ts"; +import * as datetime from "datetime"; -const UPDATE_INTERVAL_HOURS = 1; -function getAction(options: { - existingSeries: Series | null, -}) { - const { existingSeries } = options; - if (existingSeries === null) { - return "INITIAL_FETCH"; - } +const SYNC_INTERVAL_HOURS = 1; - const timeSinceLastFetch = datetime.difference( - new Date(), - existingSeries.lastFetched, - { - units: ["hours"], - } - ) +async function initialFetch(options: { id: string }) { + const fromNrk = await nrkRadio.getSerieData(options.id); + const parsed = parse.series(fromNrk); - if (timeSinceLastFetch.hours && - timeSinceLastFetch.hours > UPDATE_INTERVAL_HOURS) { - return "UPDATE"; - } + const stored = storage.write(parsed); + if (!stored) { + throw new Error(`Failed to store series ${options.id}`); + } - return "RETURN_AS_IS"; + return parsed; } +async function updateFetch(existingSeries: Series) { + const fromNrk = await nrkRadio.getSerieData(existingSeries.id); + const parsed = parse.series(fromNrk); + + const newEpisodes = parsed.episodes.filter((episode) => { + return !existingSeries.episodes.find((serieEpisode) => serieEpisode.id === episode.id); + }); + + const updated = { + ...existingSeries, + lastFetch: new Date(), + /** + * Since we don't control the API, + * we should not make assumptions about the order, + * but rather sort the episode to what we want. + */ + episodes: [...newEpisodes, ...existingSeries.episodes] + .sort((a, b) => a.date.getTime() > b.date.getTime() ? -1 : 1), + }; + + const updateSuccessful = await storage.write(updated); + if (!updateSuccessful) { + throw new Error(`Failed to update series ${existingSeries.id}`); + } + + return updated; +} + +async function getSeries(options: { id: string }): Promise { + const seriesFromStorage = await storage.read(options); + + /** + * We don't have the feed in storage, + * and we need to fetch it for the first time. + */ + if (seriesFromStorage === null) { + // TODO: schedule a job to fetch the entire backlog https://github.com/olaven/NRSS/issues/24 + return await initialFetch(options); + } + + const timeSinceLastFetch = + datetime.difference(seriesFromStorage.lastFetchedAt, new Date(), { units: ["hours"] }).hours; + + // we have the feed in storage and it's not too old + if ( + seriesFromStorage !== null && + timeSinceLastFetch !== null && + timeSinceLastFetch !== undefined && + timeSinceLastFetch <= SYNC_INTERVAL_HOURS + ) { + return seriesFromStorage; + } + + /** + * We have the feed in storage, but it's too old + * and needs to be refreshed. + */ + const updatedSeries = await updateFetch(seriesFromStorage); + return updatedSeries; +} + +export const caching = { + getSeries, +}; diff --git a/lib/parse.ts b/lib/parse.ts index 05f8292..912c65a 100644 --- a/lib/parse.ts +++ b/lib/parse.ts @@ -1,31 +1,30 @@ import { nrkRadio } from "./nrk/nrk.ts"; import { Series } from "./storage.ts"; -type NrkSerieData = Awaited> +type NrkSerieData = Awaited>; function series(nrkSeries: NrkSerieData): Series { + const imageUrl = nrkSeries.squareImage?.at(-1)?.url ?? ""; - const imageUrl = nrkSeries.squareImage?.at(-1)?.url ?? ""; - - return { - id: nrkSeries.id, - title: nrkSeries.titles.title, - subtitle: nrkSeries.titles.subtitle ?? null, - link: `https://radio.nrk.no/podkast/${nrkSeries.id}`, - imageUrl: imageUrl, - lastFetched: new Date(), - episodes: nrkSeries.episodes.map(episode => { - return { - id: episode.id, - title: episode.titles.title, - subtitle: episode.titles.subtitle ?? null, - link: episode._links.share?.href ?? "", - date: new Date(episode.date), - durationInSeconds: episode.durationInSeconds, - } - }), - } + return { + id: nrkSeries.id, + title: nrkSeries.titles.title, + subtitle: nrkSeries.titles.subtitle ?? null, + link: `https://radio.nrk.no/podkast/${nrkSeries.id}`, + imageUrl: imageUrl, + lastFetchedAt: new Date(), + episodes: nrkSeries.episodes.map((episode) => { + return { + id: episode.id, + title: episode.titles.title, + subtitle: episode.titles.subtitle ?? null, + link: episode._links.share?.href ?? "", + date: new Date(episode.date), + durationInSeconds: episode.durationInSeconds, + }; + }), + }; } export const parse = { - series, -} \ No newline at end of file + series, +}; diff --git a/lib/rss.test.ts b/lib/rss.test.ts index 6c92cdf..e399023 100644 --- a/lib/rss.test.ts +++ b/lib/rss.test.ts @@ -2,22 +2,18 @@ import { assertEquals, assertExists } from "asserts"; import { testUtils } from "./test-utils.ts"; import { rss } from "./rss.ts"; - - Deno.test("generated rss contains the series title", () => { - - const series = testUtils.generateSeries(); - const feed = rss.assembleFeed(series); - assertExists(feed) - assertEquals(feed.includes(series.title), true) + const series = testUtils.generateSeries(); + const feed = rss.assembleFeed(series); + assertExists(feed); + assertEquals(feed.includes(series.title), true); }); Deno.test("generated rss contains all episode titles", () => { - - const series = testUtils.generateSeries(); - const feed = rss.assembleFeed(series); - assertExists(feed) - series.episodes.forEach(episode => { - assertEquals(feed.includes(episode.title), true) - }) -}) \ No newline at end of file + const series = testUtils.generateSeries(); + const feed = rss.assembleFeed(series); + assertExists(feed); + series.episodes.forEach((episode) => { + assertEquals(feed.includes(episode.title), true); + }); +}); diff --git a/lib/rss.ts b/lib/rss.ts index 81047e5..6e4afd6 100644 --- a/lib/rss.ts +++ b/lib/rss.ts @@ -3,91 +3,88 @@ import { getHostName } from "../utils.ts"; import { Episode, Series } from "./storage.ts"; function assembleFeed(series: Series) { - - // Originally adapted from https://raw.githubusercontent.com/olaven/paperpod/1cde9abd3174b26e126aa74fc5a3b63fd078c0fd/packages/converter/src/rss.ts - return serialize( - declaration([ - ["version", "1.0"], - ["encoding", "UTF-8"], + // Originally adapted from https://raw.githubusercontent.com/olaven/paperpod/1cde9abd3174b26e126aa74fc5a3b63fd078c0fd/packages/converter/src/rss.ts + return serialize( + declaration([ + ["version", "1.0"], + ["encoding", "UTF-8"], + ]), + tag( + "rss", + [ + tag("channel", [ + tag("title", series.title), + tag("link", series.link), + tag("itunes:author", "NRK"), + /** + * serie.category.id does not overlap with Apple's supported categories.. + * These podcast feeds are not going to be indexed in itunes anyways, so + * a static, valid category is fine. The point is simply to pass third party + * podcast feed validation. + */ + tag("itunes:category", "", [["text", "Government"]]), + tag("itunes:owner", [ + tag("itunes:name", "NRK"), + tag("itunes:email", "nrkpodcast@nrk.no"), + ]), + tag( + "description", + series.subtitle || "", + ), + tag("ttl", "60"), //60 minutes + ...(series.imageUrl + ? [ + tag("itunes:image", "", [ + ["href", series.imageUrl], + ]), + tag("image", [ + tag("url", series.imageUrl), + tag("title", series.title), + tag("link", series.link), + ]), + ] + : []), + ...series.episodes.map((episode) => assembleEpisode({ series, episode })), ]), - tag( - "rss", - [ - tag("channel", [ - tag("title", series.title), - tag("link", series.link), - tag("itunes:author", "NRK"), - /** - * serie.category.id does not overlap with Apple's supported categories.. - * These podcast feeds are not going to be indexed in itunes anyways, so - * a static, valid category is fine. The point is simply to pass third party - * podcast feed validation. - */ - tag("itunes:category", "", [["text", "Government"]]), - tag("itunes:owner", [ - tag("itunes:name", "NRK"), - tag("itunes:email", "nrkpodcast@nrk.no"), - ]), - tag( - "description", - series.subtitle || "", - ), - tag("ttl", "60"), //60 minutes - ...(series.imageUrl - ? [ - tag("itunes:image", "", [ - ["href", series.imageUrl], - ]), - tag("image", [ - tag("url", series.imageUrl), - tag("title", series.title), - tag("link", series.link), - ]), - ] - : []), - ...series.episodes.map((episode) => assembleEpisode({ series, episode })), - ]), - ], - [ - ["version", "2.0"], - ["xmlns:itunes", "http://www.itunes.com/dtds/podcast-1.0.dtd"], - ["xmlns:content", "http://purl.org/rss/1.0/modules/content/"], - ["xmlns:podcast", "https://podcastindex.org/namespace/1.0"], - ], - ), - ); + ], + [ + ["version", "2.0"], + ["xmlns:itunes", "http://www.itunes.com/dtds/podcast-1.0.dtd"], + ["xmlns:content", "http://purl.org/rss/1.0/modules/content/"], + ["xmlns:podcast", "https://podcastindex.org/namespace/1.0"], + ], + ), + ); } -function assembleEpisode(options: { series: Series, episode: Episode }) { +function assembleEpisode(options: { series: Series; episode: Episode }) { + const { series, episode } = options; - const { series, episode } = options; + const description = episode.subtitle || ""; - const description = episode.subtitle || ""; - - return tag("item", [ - tag("title", episode.title), - tag("link", episode.link), - tag("description", description), - tag("itunes:summary", description), - tag("guid", episode.id, [["isPermaLink", "false"]]), - tag("pubDate", new Date(episode.date).toUTCString()), - tag("itunes:duration", episode.durationInSeconds.toString()), - tag("podcast:chapters", "", [ - [ - "url", - `${getHostName()}/api/feeds/${series.id}/${episode.id}/chapters`, - ], - ["type", "application/json+chapters"], - ]), - tag("enclosure", "", [ - ["url", episode.link], - ["length", episode.durationInSeconds.toString()], - ["type", "audio/mpeg3"], - ]), - ]); + return tag("item", [ + tag("title", episode.title), + tag("link", episode.link), + tag("description", description), + tag("itunes:summary", description), + tag("guid", episode.id, [["isPermaLink", "false"]]), + tag("pubDate", new Date(episode.date).toUTCString()), + tag("itunes:duration", episode.durationInSeconds.toString()), + tag("podcast:chapters", "", [ + [ + "url", + `${getHostName()}/api/feeds/${series.id}/${episode.id}/chapters`, + ], + ["type", "application/json+chapters"], + ]), + tag("enclosure", "", [ + ["url", episode.link], + ["length", episode.durationInSeconds.toString()], + ["type", "audio/mpeg3"], + ]), + ]); } - export const rss = { - assembleFeed -} \ No newline at end of file + assembleFeed, +}; diff --git a/lib/storage.test.ts b/lib/storage.test.ts index 5d62171..f02eedf 100644 --- a/lib/storage.test.ts +++ b/lib/storage.test.ts @@ -2,22 +2,20 @@ import { assertEquals } from "asserts"; import { storage } from "./storage.ts"; import { testUtils } from "./test-utils.ts"; - Deno.test("can store a series", async () => { + const series = testUtils.generateSeries(); + await storage.write(series); - const series = testUtils.generateSeries(); - await storage.write(series); + const readSeries = await storage.read(series); - const readSeries = await storage.read(series); - - assertEquals(readSeries, series); + assertEquals(readSeries, series); }); Deno.test("can retrieve a series", async () => { - const series = testUtils.generateSeries(); - await storage.write(series); + const series = testUtils.generateSeries(); + await storage.write(series); - const readSeries = await storage.read(series); + const readSeries = await storage.read(series); - assertEquals(readSeries, series); -}); \ No newline at end of file + assertEquals(readSeries, series); +}); diff --git a/lib/storage.ts b/lib/storage.ts index 75a8b9a..686a2a9 100644 --- a/lib/storage.ts +++ b/lib/storage.ts @@ -1,42 +1,42 @@ export type Episode = { - id: string, - title: string, - subtitle: string | null; - link: string, - date: Date, - durationInSeconds: number, -} + id: string; + title: string; + subtitle: string | null; + link: string; + date: Date; + durationInSeconds: number; +}; export type Series = { - id: string, - title: string, - subtitle: string | null; - link: string, - imageUrl: string, - lastFetched: Date, - episodes: Episode[], -} + id: string; + title: string; + subtitle: string | null; + link: string; + imageUrl: string; + lastFetchedAt: Date; + episodes: Episode[]; +}; const kv = await Deno.openKv(); function seriesKey(series: { id: string }) { - return ["series", series.id]; + return ["series", series.id]; } async function read(options: { id: string }): Promise { - const { id } = options; - const key = seriesKey({ id }); - const read = await kv.get(key); - return read.value; + const { id } = options; + const key = seriesKey({ id }); + const read = await kv.get(key); + return read.value; } async function write(series: Series): Promise { - const key = seriesKey(series); - const stored = await kv.set(key, series); - return stored.ok; + const key = seriesKey(series); + const stored = await kv.set(key, series); + return stored.ok; } export const storage = { - read, - write, -} \ No newline at end of file + read, + write, +}; diff --git a/lib/test-utils.ts b/lib/test-utils.ts index ad1c643..8d0ea08 100644 --- a/lib/test-utils.ts +++ b/lib/test-utils.ts @@ -1,30 +1,31 @@ -import { fakerNB_NO as faker } from "npm:@faker-js/faker" +import { fakerNB_NO as faker } from "npm:@faker-js/faker"; import { Series } from "./storage.ts"; function generateSeries(overrides: Partial = {}): Series { - return { - id: faker.word.words(1), + return { + id: faker.word.words(1), + title: faker.word.words(3), + subtitle: faker.word.words(3), + link: faker.internet.url(), + imageUrl: faker.image.url(), + lastFetchedAt: faker.date.recent(), + episodes: new Array(faker.number.int({ min: 0, max: 100 })) + .fill(null).map(() => ({ title: faker.word.words(3), subtitle: faker.word.words(3), link: faker.internet.url(), - imageUrl: faker.image.url(), - lastFetched: faker.date.recent(), - episodes: new Array(faker.number.int({ min: 0, max: 100 })) - .fill(null).map(() => ({ - title: faker.word.words(3), - subtitle: faker.word.words(3), - link: faker.internet.url(), - description: faker.lorem.paragraphs(2), - id: faker.string.uuid(), - date: faker.date.recent(), - durationInSeconds: faker.number.int({ - min: 0, max: 3600 - }), - ...overrides, - })) - } + description: faker.lorem.paragraphs(2), + id: faker.string.uuid(), + date: faker.date.recent(), + durationInSeconds: faker.number.int({ + min: 0, + max: 3600, + }), + ...overrides, + })), + }; } export const testUtils = { - generateSeries, -} \ No newline at end of file + generateSeries, +}; diff --git a/routes/api/feeds/[seriesId].ts b/routes/api/feeds/[seriesId].ts index 49ab837..fa57fdc 100644 --- a/routes/api/feeds/[seriesId].ts +++ b/routes/api/feeds/[seriesId].ts @@ -1,30 +1,16 @@ import { FreshContext } from "$fresh/server.ts"; -import { storage } from "../../../lib/storage.ts"; - - - - +import { caching } from "../../../lib/caching.ts"; +import { rss } from "../../../lib/rss.ts"; export const handler = async (_req: Request, ctx: FreshContext): Promise => { const seriesId = ctx.params.seriesId; - const storedSeries = await storage.read({ id: seriesId }); - if (storedSeries === null) { + const series = await caching.getSeries({ id: seriesId }); + const feed = rss.assembleFeed(series); - - // TODO: fetch for the first time - // const convert and store - } - - if (datetime) - - - - const feedContent = await buildFeed(seriesId); - - return new Response(feedContent, { + return new Response(feed, { headers: { "Content-Type": "application/xml", }, - }) + }); }; diff --git a/routes/api/feeds/[seriesId]/[episodeId]/chapters.ts b/routes/api/feeds/[seriesId]/[episodeId]/chapters.ts index 19eee90..1af97ac 100644 --- a/routes/api/feeds/[seriesId]/[episodeId]/chapters.ts +++ b/routes/api/feeds/[seriesId]/[episodeId]/chapters.ts @@ -1,6 +1,6 @@ import { FreshContext } from "$fresh/server.ts"; import { parse, toSeconds } from "https://esm.sh/iso8601-duration@2.1.1"; -import { nrkRadio, PodcastEpisode } from "../../../../../lib/nrk.ts"; +import { nrkRadio, PodcastEpisode } from "../../../../../lib/nrk/nrk.ts"; type Chapter = { title: string | undefined; diff --git a/routes/index.tsx b/routes/index.tsx index 1d04993..8af430f 100644 --- a/routes/index.tsx +++ b/routes/index.tsx @@ -4,8 +4,8 @@ import Footer from "../components/Footer.tsx"; import Header from "../components/Header.tsx"; import Search from "../components/Search.tsx"; import SeriesCard from "../components/SeriesCard.tsx"; -import { nrkRadio, SearchResultList } from "../lib/nrk.ts"; import { CSS, render } from "$gfm"; +import { nrkRadio, SearchResultList } from "../lib/nrk/nrk.ts"; interface HandlerData { query: string;