source: Klonkt/test/embed-resolver.test.js@ 3f32994

main
Last change on this file since 3f32994 was c52dc82, checked in by Robin Genis <roboburr@…>, 6 weeks ago

Thumbnails ontbraken: providerlijst eruit, en grote pagina-s worden gelezen i.p.v. geweigerd

Twee bugs die elkaar maskeerden, allebei van mij.

DE PROVIDERLIJST WAS HET PROBLEEM, NIET DE OPLOSSING. Een YouTube-link matchte de
hardcoded lijst, kortsloot voor oEmbed, en kwam eruit als een kaart zonder titel
en zonder thumbnail. Er werd dus niets opgeslagen. YouTube levert gewoon oEmbed
en og:image, net als de rest, dus de generieke weg doet het beter dan het
speciale geval. De lijst is weg: geen whitelist meer om te onderhouden, en of een
embed getoond mag worden is een guardian-besluit, geen kwestie van welke host het
is.

DE BODY-CAP SLOEG DE KOP ERAF. Een pagina werd geweigerd als hij groot was.
YouTube propt ~665kB inline script voor zijn og:image en sluit <head> pas op
673kB, dus we knipten net voor de tags af en hielden niets over. Nu lezen we een
pagina vanaf het BEGIN en stoppen zodra we de tags hebben (of </head>), met 1MB
als achtervang. Een normale pagina kost daardoor nog steeds een paar tientallen
kB. Het scannen kijkt alleen naar het nieuwe stuk plus wat overlap, anders wordt
een pagina van 1MB kwadratisch werk.

Echt getest, niet aangenomen: youtube.com en youtu.be leveren nu titel +
thumbnail (oembed, ~2s), linuxguides ook, boiert.eu via opengraph in 144ms.

Changed files:
src/services/EmbedResolver.js

  • providerstap verwijderd uit de keten (AudioEmbedService blijft voor de web-spelers)
  • safeHead: streamt de kop, stopt bij og:image of </head>, cap 1MB
  • safeJsonText: JSON moet heel zijn, dus daar blijft weigeren juist

src/services/ActivityPubService.js

  • self-heal 16 -> 17, en nu ook rijen die eerder niets opleverden opnieuw proberen

test/embed-resolver.test.js

  • regressietest: een videohost is geen speciaal geval en krijgt een echte kaart
  • regressietest: een grote pagina wordt afgekapt, niet geweigerd

remarks: 214 tests groen.

-robo
Co-Authored-By: Claude Opus 4.8 <noreply@…>
  • Property mode set to 100644
File size: 9.1 KB
RevLine 
[6bc2e31b]1// One embed pipeline (shaer-277): AP first (FEP semantics), then a known
2// provider, then oEmbed, then a plain link. All four end in the same shape.
3import { test } from 'node:test';
4import assert from 'node:assert/strict';
5import {
6 resolveEmbed, findOEmbedEndpoint, looksLikeAPObject, fromOEmbed,
7} from '../src/services/EmbedResolver.js';
8
9const AP_NOTE = { id: 'https://s/objects/1', type: 'Note', content: '<p>hi</p>', attributedTo: 'https://s/users/alice', url: 'https://s/@alice/1' };
10const OEMBED_PAGE = '<html><head><link rel="alternate" type="application/json+oembed" href="https://v.example/oembed?url=x&amp;f=json"><title>t</title></head></html>';
11const OEMBED_JSON = { type: 'video', title: 'A talk', author_name: 'Ada', provider_name: 'Vid', html: '<iframe src="https://v.example/e/1"></iframe>', thumbnail_url: 'https://v.example/t.jpg' };
12
13const io = (over = {}) => ({
14 getAP: async () => null,
15 getPage: async () => null,
16 getJSON: async () => null,
17 actorOf: async () => ({ name: 'Alice', handle: '@alice@s', icon: null }),
18 provider: () => null,
19 ...over,
20});
21
22test('findOEmbedEndpoint reads the json+oembed link and decodes entities', () => {
23 assert.equal(findOEmbedEndpoint(OEMBED_PAGE), 'https://v.example/oembed?url=x&f=json');
24 assert.equal(findOEmbedEndpoint('<html><head></head></html>'), null);
25 // the XML flavour is not parsed
26 assert.equal(findOEmbedEndpoint('<link rel="alternate" type="text/xml+oembed" href="https://x/o">'), null);
27});
28
29test('looksLikeAPObject accepts quotable content, rejects actors and activities', () => {
30 assert.ok(looksLikeAPObject(AP_NOTE));
31 assert.ok(looksLikeAPObject({ id: 'https://s/v/1', type: 'Video' }));
32 assert.ok(!looksLikeAPObject({ id: 'https://s/users/a', type: 'Person' }));
33 assert.ok(!looksLikeAPObject({ id: 'https://s/a/1', type: 'Create' }));
34 assert.ok(!looksLikeAPObject({ type: 'Note' })); // no id
35 assert.ok(!looksLikeAPObject(null));
36});
37
38test('an ActivityPub object resolves over AP, never over oEmbed', async () => {
39 let pageFetched = false;
40 const r = await resolveEmbed('https://s/@alice/1', io({
41 getAP: async () => AP_NOTE,
42 getPage: async () => { pageFetched = true; return OEMBED_PAGE; },
43 getJSON: async () => OEMBED_JSON,
44 provider: () => ({ provider: 'youtube', id: 'x' }), // must not win either
45 }));
46 assert.equal(r.kind, 'ap');
47 assert.equal(r.id, 'https://s/objects/1');
48 assert.equal(r.attributedTo, 'https://s/users/alice');
49 assert.equal(r.author.handle, '@alice@s');
50 assert.equal(r.url, 'https://s/@alice/1');
51 assert.ok(!pageFetched, 'AP wins before any oEmbed discovery happens');
52});
53
[c52dc82]54// Regression: a hardcoded provider list used to short-circuit here and return a
55// card with no title and no thumbnail, so a YouTube link ended up storing
56// nothing at all. There is no provider list any more; every non-fediverse URL
57// takes the generic path, which is exactly what gives it a thumbnail.
58test('a video host is not special-cased and still gets a real card', async () => {
[6bc2e31b]59 const r = await resolveEmbed('https://youtu.be/abcdefghijk', io({
[c52dc82]60 provider: () => ({ provider: 'youtube', id: 'abcdefghijk' }), // ignored on purpose
[6bc2e31b]61 getPage: async () => OEMBED_PAGE,
62 getJSON: async () => OEMBED_JSON,
63 }));
[c52dc82]64 assert.equal(r.kind, 'oembed');
65 assert.equal(r.title, 'A talk');
66 assert.ok(r.media[0].url, 'and it has a thumbnail, which the old path never produced');
[6bc2e31b]67});
68
69test('anything else goes through oEmbed discovery', async () => {
70 const r = await resolveEmbed('https://v.example/watch/1', io({
71 getPage: async () => OEMBED_PAGE,
72 getJSON: async () => OEMBED_JSON,
73 }));
74 assert.equal(r.kind, 'oembed');
75 assert.equal(r.title, 'A talk');
76 assert.equal(r.author.name, 'Ada');
77 assert.equal(r.provider, 'Vid');
78 assert.equal(r.media[0].url, 'https://v.example/t.jpg');
79 assert.ok(r.html.startsWith('<iframe'));
80});
81
82test('no oEmbed link, or a dead endpoint, still yields a usable link card', async () => {
83 const noLink = await resolveEmbed('https://plain.example/p', io({ getPage: async () => '<html></html>' }));
84 assert.equal(noLink.kind, 'link');
85 const deadEndpoint = await resolveEmbed('https://v.example/p', io({
86 getPage: async () => OEMBED_PAGE, getJSON: async () => null,
87 }));
88 assert.equal(deadEndpoint.kind, 'link');
89});
90
91test('a failing AP fetch does not abort the pipeline', async () => {
92 const r = await resolveEmbed('https://v.example/p', io({
93 getAP: async () => { throw new Error('boom'); },
94 getPage: async () => OEMBED_PAGE,
95 getJSON: async () => OEMBED_JSON,
96 }));
97 assert.equal(r.kind, 'oembed');
98});
99
100test('non-http input is refused', async () => {
101 assert.equal(await resolveEmbed('javascript:alert(1)', io()), null);
102 assert.equal(await resolveEmbed('', io()), null);
103});
104
105test('fromOEmbed keeps only an http(s) canonical url', () => {
106 const c = fromOEmbed('https://a/b', { url: 'javascript:alert(1)', title: 'x' });
107 assert.equal(c.url, 'https://a/b');
108});
109
110// The live binding: the ordering above stays, but every fetch is capped and
111// goes through safeFetch, so a URL in a post cannot make us probe internals.
112test('liveIO caps oversized bodies and never throws on a bad fetch', async () => {
113 const { liveIO } = await import('../src/services/EmbedResolver.js');
114 const calls = [];
115 const fakeFetch = async (u, o) => {
116 calls.push([u, o.headers.Accept]);
117 if (u.includes('huge')) return { ok: true, headers: { get: () => String(9_000_000) }, text: async () => 'x' };
118 if (u.includes('boom')) throw new Error('refused');
119 return { ok: true, headers: { get: () => '10' }, text: async () => '{"type":"Note","id":"https://s/1"}' };
120 };
121 const io = liveIO({ safeFetch: fakeFetch, detectProvider: () => null });
[c52dc82]122 // A JSON payload must parse whole, so an oversized one is refused outright.
123 assert.equal(await io.getJSON('https://x/huge'), null, 'oversized JSON refused');
[6bc2e31b]124 assert.equal(await io.getAP('https://x/boom'), null, 'a refused fetch is not an error');
125 assert.deepEqual(await io.getAP('https://x/ok'), { type: 'Note', id: 'https://s/1' });
126 assert.ok(calls.some((c) => c[1].includes('activity+json')), 'AP asks for activity+json');
127});
[0101d0a]128
129// OpenGraph: the one that actually carries link previews on the open web.
130// oEmbed is richer, but most sites simply do not ship it, which is why cards
131// stayed empty until this fallback existed.
132const OG_PAGE = '<html><head><meta property="og:title" content="Linux f&amp;uuml;r Einsteiger">'
133 + '<meta property="og:site_name" content="Linux Guides">'
134 + '<meta property="og:image" content="https://lg.example/tux.png"></head></html>';
135
136test('findOpenGraph reads og:image/title/site and decodes entities', async () => {
137 const { findOpenGraph } = await import('../src/services/EmbedResolver.js');
138 const og = findOpenGraph(OG_PAGE);
139 assert.equal(og.image, 'https://lg.example/tux.png');
140 assert.equal(og.site, 'Linux Guides');
141 assert.ok(og.title.startsWith('Linux f'));
142 assert.equal(findOpenGraph('<html><head><title>x</title></head></html>'), null);
143 assert.equal(findOpenGraph(null), null);
144});
145
146test('findOpenGraph falls back to twitter:image and refuses a non-http image', async () => {
147 const { findOpenGraph } = await import('../src/services/EmbedResolver.js');
148 const tw = findOpenGraph('<meta name="twitter:image" content="https://x/y.png"><meta name="twitter:title" content="T">');
149 assert.equal(tw.image, 'https://x/y.png');
150 const bad = findOpenGraph('<meta property="og:image" content="javascript:alert(1)"><meta property="og:title" content="T">');
151 assert.equal(bad.image, null, 'a non-http image is dropped, the title survives');
152 assert.equal(bad.title, 'T');
153});
154
155test('oEmbed still wins over OpenGraph when a page offers both', async () => {
156 const both = OEMBED_PAGE.replace('</head>', '<meta property="og:title" content="OG"></head>');
157 const r = await resolveEmbed('https://v.example/1', io({
158 getPage: async () => both, getJSON: async () => OEMBED_JSON,
159 }));
160 assert.equal(r.kind, 'oembed');
161 assert.equal(r.title, 'A talk');
162});
163
164test('a page with only OpenGraph yields a thumbnail card', async () => {
165 const r = await resolveEmbed('https://lg.example/artikel', io({ getPage: async () => OG_PAGE }));
166 assert.equal(r.kind, 'opengraph');
167 assert.equal(r.media[0].url, 'https://lg.example/tux.png');
168 assert.equal(r.provider, 'Linux Guides');
169 assert.equal(r.url, 'https://lg.example/artikel');
170});
[c52dc82]171
172// Regression, the one that kept YouTube blank: a page is read from the START and
173// cut off, never refused for being large. Refusing it meant no thumbnail at all
174// for exactly the sites people share most.
175test('a huge page is truncated, not rejected', async () => {
176 const { liveIO } = await import('../src/services/EmbedResolver.js');
177 const big = '<html><head>' + 'x'.repeat(5000) + '<meta property="og:title" content="T">'
178 + '<meta property="og:image" content="https://x/i.png"></head></html>';
179 const fakeFetch = async () => ({ ok: true, headers: { get: () => String(9_000_000) }, text: async () => big });
180 const io = liveIO({ safeFetch: fakeFetch, detectProvider: () => null });
181 const page = await io.getPage('https://x/huge');
182 assert.ok(page && page.includes('og:image'), 'the head survives the cap');
183});
Note: See TracBrowser for help on using the repository browser.