From e6d71710e5dcd9a937efd6517249f5d2170300b9 Mon Sep 17 00:00:00 2001 From: olaven Date: Thu, 19 Sep 2024 19:29:17 +0200 Subject: [PATCH 1/5] Ensure episodes are within Deno KV's size limitations Not the most elegant solution. But it works at the expense of old episodes. I should think of something better at some point. Left some ideas in the code for future reference. --- lib/caching.ts | 34 +++++++++++++++++++++++++++++++--- lib/utils.ts | 2 -- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/lib/caching.ts b/lib/caching.ts index b51afa0..acc16c5 100644 --- a/lib/caching.ts +++ b/lib/caching.ts @@ -2,8 +2,10 @@ import { nrkRadio } from "./nrk/nrk.ts"; import { Series, storage } from "./storage.ts"; import * as datetime from "$std/datetime/mod.ts"; import { Episode } from "./storage.ts"; +import { Buffer } from "node:buffer"; const SYNC_INTERVAL_HOURS = 1; +const DENO_KV_MAX_BYTES = 65_536; async function initialFetch(options: { id: string }): Promise { const series = await nrkRadio.getSeries(options.id); @@ -23,6 +25,7 @@ type UpdatedSeries = { lastFetch: Date; episodes: Episode[]; } & Series; + async function updateFetch(existingSeries: Series): Promise { const series = await nrkRadio.getSeries(existingSeries.id); if (!series) { @@ -44,19 +47,44 @@ async function updateFetch(existingSeries: Series): Promise a.date.getTime() > b.date.getTime() ? -1 : 1); - const updated = { + const withNewEpisodes = { ...existingSeries, lastFetch: new Date(), episodes: episodesSortedDescending, }; - const updateSuccessful = await storage.writeSeries(updated); + /** + * The KV store has a limit of 64kb per value. + * A pragmatic (not perfect) solution is to trim the series + * down until we're within the limit. + * + * Other ideas for the future: + * - splitting the series into multiple keys + * - using a different storage solution + */ + const trimmed = trimSeriesToSize(withNewEpisodes, DENO_KV_MAX_BYTES); + + const updateSuccessful = await storage.writeSeries(trimmed); if (!updateSuccessful) { console.log(`Failed to update series ${existingSeries.id}`); return null; } - return updated; + return trimmed; +} + +function trimSeriesToSize(series: Series, bytes: number): Series { + const currentBytes = Buffer.byteLength(JSON.stringify(series)); + if (currentBytes <= bytes) { + return series; + } + + const trimmed = { + ...series, + episodes: series.episodes.slice(0, -1), + }; + + return trimSeriesToSize(trimmed, bytes); } function getTimeSinceLastFetch(inputDate: Date, againstDate = new Date()): number | null { diff --git a/lib/utils.ts b/lib/utils.ts index 370f50e..55e6d6a 100644 --- a/lib/utils.ts +++ b/lib/utils.ts @@ -6,10 +6,8 @@ export function getHostUrl() { if (deploymentId) { return `https://nrss-${deploymentId}.deno.dev`; } else if (tunnelUrl) { - console.debug(`Using tunnel URL: ${tunnelUrl}`); return tunnelUrl; } else { - console.debug(`Assuming localhost`); return "http://localhost:8000"; } } -- 2.40.1 From 4912829440747b686f6325e26db61defb5711b28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Olav=20Sundf=C3=B8r?= Date: Thu, 19 Sep 2024 19:31:53 +0200 Subject: [PATCH 2/5] Ensure episodes are within Deno KV's size limitations (#37) Not the most elegant solution. But it works at the expense of old episodes. I should think of something better at some point. Left some ideas in the code for future reference. --- lib/caching.ts | 34 +++++++++++++++++++++++++++++++--- lib/utils.ts | 2 -- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/lib/caching.ts b/lib/caching.ts index b51afa0..acc16c5 100644 --- a/lib/caching.ts +++ b/lib/caching.ts @@ -2,8 +2,10 @@ import { nrkRadio } from "./nrk/nrk.ts"; import { Series, storage } from "./storage.ts"; import * as datetime from "$std/datetime/mod.ts"; import { Episode } from "./storage.ts"; +import { Buffer } from "node:buffer"; const SYNC_INTERVAL_HOURS = 1; +const DENO_KV_MAX_BYTES = 65_536; async function initialFetch(options: { id: string }): Promise { const series = await nrkRadio.getSeries(options.id); @@ -23,6 +25,7 @@ type UpdatedSeries = { lastFetch: Date; episodes: Episode[]; } & Series; + async function updateFetch(existingSeries: Series): Promise { const series = await nrkRadio.getSeries(existingSeries.id); if (!series) { @@ -44,19 +47,44 @@ async function updateFetch(existingSeries: Series): Promise a.date.getTime() > b.date.getTime() ? -1 : 1); - const updated = { + const withNewEpisodes = { ...existingSeries, lastFetch: new Date(), episodes: episodesSortedDescending, }; - const updateSuccessful = await storage.writeSeries(updated); + /** + * The KV store has a limit of 64kb per value. + * A pragmatic (not perfect) solution is to trim the series + * down until we're within the limit. + * + * Other ideas for the future: + * - splitting the series into multiple keys + * - using a different storage solution + */ + const trimmed = trimSeriesToSize(withNewEpisodes, DENO_KV_MAX_BYTES); + + const updateSuccessful = await storage.writeSeries(trimmed); if (!updateSuccessful) { console.log(`Failed to update series ${existingSeries.id}`); return null; } - return updated; + return trimmed; +} + +function trimSeriesToSize(series: Series, bytes: number): Series { + const currentBytes = Buffer.byteLength(JSON.stringify(series)); + if (currentBytes <= bytes) { + return series; + } + + const trimmed = { + ...series, + episodes: series.episodes.slice(0, -1), + }; + + return trimSeriesToSize(trimmed, bytes); } function getTimeSinceLastFetch(inputDate: Date, againstDate = new Date()): number | null { diff --git a/lib/utils.ts b/lib/utils.ts index 370f50e..55e6d6a 100644 --- a/lib/utils.ts +++ b/lib/utils.ts @@ -6,10 +6,8 @@ export function getHostUrl() { if (deploymentId) { return `https://nrss-${deploymentId}.deno.dev`; } else if (tunnelUrl) { - console.debug(`Using tunnel URL: ${tunnelUrl}`); return tunnelUrl; } else { - console.debug(`Assuming localhost`); return "http://localhost:8000"; } } -- 2.40.1 From d491f129c053ab946fb46b5bc22ac7b8af29a441 Mon Sep 17 00:00:00 2001 From: olaven Date: Mon, 23 Sep 2024 17:29:52 +0200 Subject: [PATCH 3/5] Trim and anonymize idempotency key --- lib/vipps/vipps.ts | 4 +++- todo.md | 12 ++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 todo.md diff --git a/lib/vipps/vipps.ts b/lib/vipps/vipps.ts index 47715a6..33cdeb3 100644 --- a/lib/vipps/vipps.ts +++ b/lib/vipps/vipps.ts @@ -1,5 +1,6 @@ // deno-lint-ignore-file ban-ts-comment import "jsr:@std/dotenv/load"; +import { encodeHex } from "jsr:@std/encoding/hex"; import { getHostUrl } from "../utils.ts"; import { STATUS_CODE } from "$fresh/server.ts"; @@ -50,7 +51,8 @@ export const createAgreement = async function (email: string) { // @ts-ignore headers: { authorization: `Bearer ${token}`, - "Idempotency-Key": `${email}-${Date.now()}`, + // anonymized and trimmed to not be too long (then it fails) + "Idempotency-Key": `${encodeHex(email).slice(0, 10)}-${Date.now()}`, ...standardVippsHeaders, }, body: JSON.stringify({ diff --git a/todo.md b/todo.md new file mode 100644 index 0000000..fd53b0d --- /dev/null +++ b/todo.md @@ -0,0 +1,12 @@ +# Bugs +- [ ] enter on sponsor input reloads the page + - [ ] reproduce + - [ ] fix + - [ ] report back to user (MFA email) +- [ ] user (MFA email) gets an error with one of his emails (see mail thread) + - [x] reproduce + - [x] fix + - [ ] report back to user +- [ ] there are more users registered as paying users than there are payments received + - [ ] are these people supposed to still be subscribers? + - [ ] how can this have happenned? \ No newline at end of file -- 2.40.1 From 3d07c813f77bfb96096881c3c00ab73f708b5b31 Mon Sep 17 00:00:00 2001 From: olaven Date: Mon, 23 Sep 2024 17:38:29 +0200 Subject: [PATCH 4/5] Don't catch submit events in Vipps button --- islands/DonationSection.tsx | 7 ++++++- todo.md | 4 ++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/islands/DonationSection.tsx b/islands/DonationSection.tsx index 84dec1d..3f98708 100644 --- a/islands/DonationSection.tsx +++ b/islands/DonationSection.tsx @@ -30,7 +30,12 @@ export const DonationSection = function () {

-
+ { + e.preventDefault(); + }} + > Date: Mon, 23 Sep 2024 17:41:21 +0200 Subject: [PATCH 5/5] Formatting --- todo.md | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/todo.md b/todo.md index 02476ab..852bf6b 100644 --- a/todo.md +++ b/todo.md @@ -1,12 +1,13 @@ -# Bugs -- [ ] enter on sponsor input reloads the page - - [x] reproduce - - [x] fix +# Bugs + +- [ ] enter on sponsor input reloads the page + - [x] reproduce + - [x] fix - [ ] report back to user (MFA email) - [ ] user (MFA email) gets an error with one of his emails (see mail thread) - - [x] reproduce - - [x] fix - - [ ] report back to user -- [ ] there are more users registered as paying users than there are payments received - - [ ] are these people supposed to still be subscribers? - - [ ] how can this have happenned? \ No newline at end of file + - [x] reproduce + - [x] fix + - [ ] report back to user +- [ ] there are more users registered as paying users than there are payments received + - [ ] are these people supposed to still be subscribers? + - [ ] how can this have happenned? -- 2.40.1