olaven
/
nrss
1
0
Fork 0
Code Issues 6 Pull Requests Packages Projects Releases Wiki Activity

Inital implementation

This commit is contained in:
olaven 2024-04-07 11:03:59 +02:00
parent b632ea4f95
commit bfadcae817
12 changed files with 254 additions and 222 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
components/SeriesCard.tsx
View File

@ -1,5 +1,5 @@
import { SearchResult } from "../lib/nrk.ts";
import CopyButton from "../islands/CopyButton.tsx";
import { SearchResult } from "../lib/nrk/nrk.ts";
export default function SeriesCard(props: { serie: SearchResult; origin: string }) {
const feedUrl = new URL(`/api/feeds/${props.serie.seriesId}`, props.origin);

99
lib/caching.ts
View File

@ -1,28 +1,83 @@
import * as datetime from "datetime"
import { Series } from "./storage.ts"
import { nrkRadio } from "./nrk/nrk.ts";
import { parse } from "./parse.ts";
import { Series, storage } from "./storage.ts";
import * as datetime from "datetime";
const UPDATE_INTERVAL_HOURS = 1;
function getAction(options: {
existingSeries: Series | null,
}) {
const { existingSeries } = options;
if (existingSeries === null) {
return "INITIAL_FETCH";
}
const SYNC_INTERVAL_HOURS = 1;
const timeSinceLastFetch = datetime.difference(
new Date(),
existingSeries.lastFetched,
{
units: ["hours"],
}
)
async function initialFetch(options: { id: string }) {
const fromNrk = await nrkRadio.getSerieData(options.id);
const parsed = parse.series(fromNrk);
if (timeSinceLastFetch.hours &&
timeSinceLastFetch.hours > UPDATE_INTERVAL_HOURS) {
return "UPDATE";
}
const stored = storage.write(parsed);
if (!stored) {
throw new Error(`Failed to store series ${options.id}`);
}
return "RETURN_AS_IS";
return parsed;
}
async function updateFetch(existingSeries: Series) {
const fromNrk = await nrkRadio.getSerieData(existingSeries.id);
const parsed = parse.series(fromNrk);
const newEpisodes = parsed.episodes.filter((episode) => {
return !existingSeries.episodes.find((serieEpisode) => serieEpisode.id === episode.id);
});
const updated = {
...existingSeries,
lastFetch: new Date(),
/**
* Since we don't control the API,
* we should not make assumptions about the order,
* but rather sort the episode to what we want.
*/
episodes: [...newEpisodes, ...existingSeries.episodes]
.sort((a, b) => a.date.getTime() > b.date.getTime() ? -1 : 1),
};
const updateSuccessful = await storage.write(updated);
if (!updateSuccessful) {
throw new Error(`Failed to update series ${existingSeries.id}`);
}
return updated;
}
async function getSeries(options: { id: string }): Promise<Series> {
const seriesFromStorage = await storage.read(options);
/**
* We don't have the feed in storage,
* and we need to fetch it for the first time.
*/
if (seriesFromStorage === null) {
// TODO: schedule a job to fetch the entire backlog https://github.com/olaven/NRSS/issues/24
return await initialFetch(options);
}
const timeSinceLastFetch =
datetime.difference(seriesFromStorage.lastFetchedAt, new Date(), { units: ["hours"] }).hours;
// we have the feed in storage and it's not too old
if (
seriesFromStorage !== null &&
timeSinceLastFetch !== null &&
timeSinceLastFetch !== undefined &&
timeSinceLastFetch <= SYNC_INTERVAL_HOURS
) {
return seriesFromStorage;
}
/**
* We have the feed in storage, but it's too old
* and needs to be refreshed.
*/
const updatedSeries = await updateFetch(seriesFromStorage);
return updatedSeries;
}
export const caching = {
getSeries,
};

45
lib/parse.ts
View File

@ -1,31 +1,30 @@
import { nrkRadio } from "./nrk/nrk.ts";
import { Series } from "./storage.ts";
type NrkSerieData = Awaited<ReturnType<typeof nrkRadio.getSerieData>>
type NrkSerieData = Awaited<ReturnType<typeof nrkRadio.getSerieData>>;
function series(nrkSeries: NrkSerieData): Series {
const imageUrl = nrkSeries.squareImage?.at(-1)?.url ?? "";
const imageUrl = nrkSeries.squareImage?.at(-1)?.url ?? "";
return {
id: nrkSeries.id,
title: nrkSeries.titles.title,
subtitle: nrkSeries.titles.subtitle ?? null,
link: `https://radio.nrk.no/podkast/${nrkSeries.id}`,
imageUrl: imageUrl,
lastFetched: new Date(),
episodes: nrkSeries.episodes.map(episode => {
return {
id: episode.id,
title: episode.titles.title,
subtitle: episode.titles.subtitle ?? null,
link: episode._links.share?.href ?? "",
date: new Date(episode.date),
durationInSeconds: episode.durationInSeconds,
}
}),
}
return {
id: nrkSeries.id,
title: nrkSeries.titles.title,
subtitle: nrkSeries.titles.subtitle ?? null,
link: `https://radio.nrk.no/podkast/${nrkSeries.id}`,
imageUrl: imageUrl,
lastFetchedAt: new Date(),
episodes: nrkSeries.episodes.map((episode) => {
return {
id: episode.id,
title: episode.titles.title,
subtitle: episode.titles.subtitle ?? null,
link: episode._links.share?.href ?? "",
date: new Date(episode.date),
durationInSeconds: episode.durationInSeconds,
};
}),
};
}
export const parse = {
series,
}
series,
};

26
lib/rss.test.ts
View File

@ -2,22 +2,18 @@ import { assertEquals, assertExists } from "asserts";
import { testUtils } from "./test-utils.ts";
import { rss } from "./rss.ts";
Deno.test("generated rss contains the series title", () => {
const series = testUtils.generateSeries();
const feed = rss.assembleFeed(series);
assertExists(feed)
assertEquals(feed.includes(series.title), true)
const series = testUtils.generateSeries();
const feed = rss.assembleFeed(series);
assertExists(feed);
assertEquals(feed.includes(series.title), true);
});
Deno.test("generated rss contains all episode titles", () => {
const series = testUtils.generateSeries();
const feed = rss.assembleFeed(series);
assertExists(feed)
series.episodes.forEach(episode => {
assertEquals(feed.includes(episode.title), true)
})
})
const series = testUtils.generateSeries();
const feed = rss.assembleFeed(series);
assertExists(feed);
series.episodes.forEach((episode) => {
assertEquals(feed.includes(episode.title), true);
});
});

157
lib/rss.ts
View File

@ -3,91 +3,88 @@ import { getHostName } from "../utils.ts";
import { Episode, Series } from "./storage.ts";
function assembleFeed(series: Series) {
// Originally adapted from https://raw.githubusercontent.com/olaven/paperpod/1cde9abd3174b26e126aa74fc5a3b63fd078c0fd/packages/converter/src/rss.ts
return serialize(
declaration([
["version", "1.0"],
["encoding", "UTF-8"],
// Originally adapted from https://raw.githubusercontent.com/olaven/paperpod/1cde9abd3174b26e126aa74fc5a3b63fd078c0fd/packages/converter/src/rss.ts
return serialize(
declaration([
["version", "1.0"],
["encoding", "UTF-8"],
]),
tag(
"rss",
[
tag("channel", [
tag("title", series.title),
tag("link", series.link),
tag("itunes:author", "NRK"),
/**
* serie.category.id does not overlap with Apple's supported categories..
* These podcast feeds are not going to be indexed in itunes anyways, so
* a static, valid category is fine. The point is simply to pass third party
* podcast feed validation.
*/
tag("itunes:category", "", [["text", "Government"]]),
tag("itunes:owner", [
tag("itunes:name", "NRK"),
tag("itunes:email", "nrkpodcast@nrk.no"),
]),
tag(
"description",
series.subtitle || "",
),
tag("ttl", "60"), //60 minutes
...(series.imageUrl
? [
tag("itunes:image", "", [
["href", series.imageUrl],
]),
tag("image", [
tag("url", series.imageUrl),
tag("title", series.title),
tag("link", series.link),
]),
]
: []),
...series.episodes.map((episode) => assembleEpisode({ series, episode })),
]),
tag(
"rss",
[
tag("channel", [
tag("title", series.title),
tag("link", series.link),
tag("itunes:author", "NRK"),
/**
* serie.category.id does not overlap with Apple's supported categories..
* These podcast feeds are not going to be indexed in itunes anyways, so
* a static, valid category is fine. The point is simply to pass third party
* podcast feed validation.
*/
tag("itunes:category", "", [["text", "Government"]]),
tag("itunes:owner", [
tag("itunes:name", "NRK"),
tag("itunes:email", "nrkpodcast@nrk.no"),
]),
tag(
"description",
series.subtitle || "",
),
tag("ttl", "60"), //60 minutes
...(series.imageUrl
? [
tag("itunes:image", "", [
["href", series.imageUrl],
]),
tag("image", [
tag("url", series.imageUrl),
tag("title", series.title),
tag("link", series.link),
]),
]
: []),
...series.episodes.map((episode) => assembleEpisode({ series, episode })),
]),
],
[
["version", "2.0"],
["xmlns:itunes", "http://www.itunes.com/dtds/podcast-1.0.dtd"],
["xmlns:content", "http://purl.org/rss/1.0/modules/content/"],
["xmlns:podcast", "https://podcastindex.org/namespace/1.0"],
],
),
);
],
[
["version", "2.0"],
["xmlns:itunes", "http://www.itunes.com/dtds/podcast-1.0.dtd"],
["xmlns:content", "http://purl.org/rss/1.0/modules/content/"],
["xmlns:podcast", "https://podcastindex.org/namespace/1.0"],
],
),
);
}
function assembleEpisode(options: { series: Series, episode: Episode }) {
function assembleEpisode(options: { series: Series; episode: Episode }) {
const { series, episode } = options;
const { series, episode } = options;
const description = episode.subtitle || "";
const description = episode.subtitle || "";
return tag("item", [
tag("title", episode.title),
tag("link", episode.link),
tag("description", description),
tag("itunes:summary", description),
tag("guid", episode.id, [["isPermaLink", "false"]]),
tag("pubDate", new Date(episode.date).toUTCString()),
tag("itunes:duration", episode.durationInSeconds.toString()),
tag("podcast:chapters", "", [
[
"url",
`${getHostName()}/api/feeds/${series.id}/${episode.id}/chapters`,
],
["type", "application/json+chapters"],
]),
tag("enclosure", "", [
["url", episode.link],
["length", episode.durationInSeconds.toString()],
["type", "audio/mpeg3"],
]),
]);
return tag("item", [
tag("title", episode.title),
tag("link", episode.link),
tag("description", description),
tag("itunes:summary", description),
tag("guid", episode.id, [["isPermaLink", "false"]]),
tag("pubDate", new Date(episode.date).toUTCString()),
tag("itunes:duration", episode.durationInSeconds.toString()),
tag("podcast:chapters", "", [
[
"url",
`${getHostName()}/api/feeds/${series.id}/${episode.id}/chapters`,
],
["type", "application/json+chapters"],
]),
tag("enclosure", "", [
["url", episode.link],
["length", episode.durationInSeconds.toString()],
["type", "audio/mpeg3"],
]),
]);
}
export const rss = {
assembleFeed
}
assembleFeed,
};

18
lib/storage.test.ts
View File

@ -2,22 +2,20 @@ import { assertEquals } from "asserts";
import { storage } from "./storage.ts";
import { testUtils } from "./test-utils.ts";
Deno.test("can store a series", async () => {
const series = testUtils.generateSeries();
await storage.write(series);
const series = testUtils.generateSeries();
await storage.write(series);
const readSeries = await storage.read(series);
const readSeries = await storage.read(series);
assertEquals(readSeries, series);
assertEquals(readSeries, series);
});
Deno.test("can retrieve a series", async () => {
const series = testUtils.generateSeries();
await storage.write(series);
const series = testUtils.generateSeries();
await storage.write(series);
const readSeries = await storage.read(series);
const readSeries = await storage.read(series);
assertEquals(readSeries, series);
assertEquals(readSeries, series);
});

52
lib/storage.ts
View File

@ -1,42 +1,42 @@
export type Episode = {
id: string,
title: string,
subtitle: string | null;
link: string,
date: Date,
durationInSeconds: number,
}
id: string;
title: string;
subtitle: string | null;
link: string;
date: Date;
durationInSeconds: number;
};
export type Series = {
id: string,
title: string,
subtitle: string | null;
link: string,
imageUrl: string,
lastFetched: Date,
episodes: Episode[],
}
id: string;
title: string;
subtitle: string | null;
link: string;
imageUrl: string;
lastFetchedAt: Date;
episodes: Episode[];
};
const kv = await Deno.openKv();
function seriesKey(series: { id: string }) {
return ["series", series.id];
return ["series", series.id];
}
async function read(options: { id: string }): Promise<Series | null> {
const { id } = options;
const key = seriesKey({ id });
const read = await kv.get<Series>(key);
return read.value;
const { id } = options;
const key = seriesKey({ id });
const read = await kv.get<Series>(key);
return read.value;
}
async function write(series: Series): Promise<boolean> {
const key = seriesKey(series);
const stored = await kv.set(key, series);
return stored.ok;
const key = seriesKey(series);
const stored = await kv.set(key, series);
return stored.ok;
}
export const storage = {
read,
write,
}
read,
write,
};

43
lib/test-utils.ts
View File

@ -1,30 +1,31 @@
import { fakerNB_NO as faker } from "npm:@faker-js/faker"
import { fakerNB_NO as faker } from "npm:@faker-js/faker";
import { Series } from "./storage.ts";
function generateSeries(overrides: Partial<Series> = {}): Series {
return {
id: faker.word.words(1),
return {
id: faker.word.words(1),
title: faker.word.words(3),
subtitle: faker.word.words(3),
link: faker.internet.url(),
imageUrl: faker.image.url(),
lastFetchedAt: faker.date.recent(),
episodes: new Array(faker.number.int({ min: 0, max: 100 }))
.fill(null).map(() => ({
title: faker.word.words(3),
subtitle: faker.word.words(3),
link: faker.internet.url(),
imageUrl: faker.image.url(),
lastFetched: faker.date.recent(),
episodes: new Array(faker.number.int({ min: 0, max: 100 }))
.fill(null).map(() => ({
title: faker.word.words(3),
subtitle: faker.word.words(3),
link: faker.internet.url(),
description: faker.lorem.paragraphs(2),
id: faker.string.uuid(),
date: faker.date.recent(),
durationInSeconds: faker.number.int({
min: 0, max: 3600
}),
...overrides,
}))
}
description: faker.lorem.paragraphs(2),
id: faker.string.uuid(),
date: faker.date.recent(),
durationInSeconds: faker.number.int({
min: 0,
max: 3600,
}),
...overrides,
})),
};
}
export const testUtils = {
generateSeries,
}
generateSeries,
};

26
routes/api/feeds/[seriesId].ts
View File

@ -1,30 +1,16 @@
import { FreshContext } from "$fresh/server.ts";
import { storage } from "../../../lib/storage.ts";
import { caching } from "../../../lib/caching.ts";
import { rss } from "../../../lib/rss.ts";
export const handler = async (_req: Request, ctx: FreshContext): Promise<Response> => {
const seriesId = ctx.params.seriesId;
const storedSeries = await storage.read({ id: seriesId });
if (storedSeries === null) {
const series = await caching.getSeries({ id: seriesId });
const feed = rss.assembleFeed(series);
// TODO: fetch for the first time
// const convert and store
}
if (datetime)
const feedContent = await buildFeed(seriesId);
return new Response(feedContent, {
return new Response(feed, {
headers: {
"Content-Type": "application/xml",
},
})
});
};

2
routes/api/feeds/[seriesId]/[episodeId]/chapters.ts
View File

@ -1,6 +1,6 @@
import { FreshContext } from "$fresh/server.ts";
import { parse, toSeconds } from "https://esm.sh/iso8601-duration@2.1.1";
import { nrkRadio, PodcastEpisode } from "../../../../../lib/nrk.ts";
import { nrkRadio, PodcastEpisode } from "../../../../../lib/nrk/nrk.ts";
type Chapter = {
title: string | undefined;

2
routes/index.tsx
View File

@ -4,8 +4,8 @@ import Footer from "../components/Footer.tsx";
import Header from "../components/Header.tsx";
import Search from "../components/Search.tsx";
import SeriesCard from "../components/SeriesCard.tsx";
import { nrkRadio, SearchResultList } from "../lib/nrk.ts";
import { CSS, render } from "$gfm";
import { nrkRadio, SearchResultList } from "../lib/nrk/nrk.ts";
interface HandlerData {
query: string;