source: Klonkt/test/embed-resolver.test.js@ 3ccca13

main
Last change on this file since 3ccca13 was bdb78e4, checked in by Robin Genis <roboburr@…>, 6 weeks ago

Publiek oEmbed-register erbij: YouTube verstopt zijn tags voor een server

Waarom het op mijn machine werkte en op de VPS niet: YouTube serveert een
datacenter-IP een uitgeklede pagina. Titel leeg, nul meta property-tags, geen
oEmbed-link. Hun oEmbed-API antwoordt daar wel gewoon (status 200, echte titel).
Discovery via de pagina faalt dus precies waar het het meest uitmaakt.

Het publieke providerregister van oembed.com dicht dat gat, en dat is iets anders
dan de whitelist die we net weggehaald hebben: die lijst onderhouden wij niet, we
lezen er een die gepubliceerd wordt. Valt oembed.com weg, dan vallen we terug op
paginadiscovery en werkt alles gewoon minder slim.

Het register komt VOOR de paginafetch: het is een match in het geheugen plus een
klein API-verzoek, en scheelt dus ook de download van een pagina van een MB.
Eenmaal opgehaald blijft de lijst een dag staan; een stale lijst is beter dan
geen lijst.

Changed files:
src/services/EmbedResolver.js

  • matchProviderEndpoint (wildcard-schemes, host-fallback, {format} invullen)
  • oembedRequestUrl; registry-stap voor de pagina; liveIO cachet de lijst 24u

src/services/ActivityPubService.js

  • SELFHEAL_VERSION 19 -> 20

test/embed-resolver.test.js

  • 4 tests: schemes en host-fallback, url-opbouw, register wint van de pagina (en bespaart de download), en een onbereikbaar register valt netjes terug

remarks: 219 tests groen.

-robo
Co-Authored-By: Claude Opus 4.8 <noreply@…>
  • Property mode set to 100644
File size: 12.5 KB
Line 
1// One embed pipeline (shaer-277): AP first (FEP semantics), then a known
2// provider, then oEmbed, then a plain link. All four end in the same shape.
3import { test } from 'node:test';
4import assert from 'node:assert/strict';
5import {
6 resolveEmbed, findOEmbedEndpoint, looksLikeAPObject, fromOEmbed,
7} from '../src/services/EmbedResolver.js';
8
9const AP_NOTE = { id: 'https://s/objects/1', type: 'Note', content: '<p>hi</p>', attributedTo: 'https://s/users/alice', url: 'https://s/@alice/1' };
10const OEMBED_PAGE = '<html><head><link rel="alternate" type="application/json+oembed" href="https://v.example/oembed?url=x&amp;f=json"><title>t</title></head></html>';
11const OEMBED_JSON = { type: 'video', title: 'A talk', author_name: 'Ada', provider_name: 'Vid', html: '<iframe src="https://v.example/e/1"></iframe>', thumbnail_url: 'https://v.example/t.jpg' };
12
13const io = (over = {}) => ({
14 getAP: async () => null,
15 getPage: async () => null,
16 getJSON: async () => null,
17 actorOf: async () => ({ name: 'Alice', handle: '@alice@s', icon: null }),
18 provider: () => null,
19 ...over,
20});
21
22test('findOEmbedEndpoint reads the json+oembed link and decodes entities', () => {
23 assert.equal(findOEmbedEndpoint(OEMBED_PAGE), 'https://v.example/oembed?url=x&f=json');
24 assert.equal(findOEmbedEndpoint('<html><head></head></html>'), null);
25 // the XML flavour is not parsed
26 assert.equal(findOEmbedEndpoint('<link rel="alternate" type="text/xml+oembed" href="https://x/o">'), null);
27});
28
29test('looksLikeAPObject accepts quotable content, rejects actors and activities', () => {
30 assert.ok(looksLikeAPObject(AP_NOTE));
31 assert.ok(looksLikeAPObject({ id: 'https://s/v/1', type: 'Video' }));
32 assert.ok(!looksLikeAPObject({ id: 'https://s/users/a', type: 'Person' }));
33 assert.ok(!looksLikeAPObject({ id: 'https://s/a/1', type: 'Create' }));
34 assert.ok(!looksLikeAPObject({ type: 'Note' })); // no id
35 assert.ok(!looksLikeAPObject(null));
36});
37
38test('an ActivityPub object resolves over AP, never over oEmbed', async () => {
39 let pageFetched = false;
40 const r = await resolveEmbed('https://s/@alice/1', io({
41 getAP: async () => AP_NOTE,
42 getPage: async () => { pageFetched = true; return OEMBED_PAGE; },
43 getJSON: async () => OEMBED_JSON,
44 provider: () => ({ provider: 'youtube', id: 'x' }), // must not win either
45 }));
46 assert.equal(r.kind, 'ap');
47 assert.equal(r.id, 'https://s/objects/1');
48 assert.equal(r.attributedTo, 'https://s/users/alice');
49 assert.equal(r.author.handle, '@alice@s');
50 assert.equal(r.url, 'https://s/@alice/1');
51 assert.ok(!pageFetched, 'AP wins before any oEmbed discovery happens');
52});
53
54// Regression: a hardcoded provider list used to short-circuit here and return a
55// card with no title and no thumbnail, so a YouTube link ended up storing
56// nothing at all. There is no provider list any more; every non-fediverse URL
57// takes the generic path, which is exactly what gives it a thumbnail.
58test('a video host is not special-cased and still gets a real card', async () => {
59 const r = await resolveEmbed('https://youtu.be/abcdefghijk', io({
60 provider: () => ({ provider: 'youtube', id: 'abcdefghijk' }), // ignored on purpose
61 getPage: async () => OEMBED_PAGE,
62 getJSON: async () => OEMBED_JSON,
63 }));
64 assert.equal(r.kind, 'oembed');
65 assert.equal(r.title, 'A talk');
66 assert.ok(r.media[0].url, 'and it has a thumbnail, which the old path never produced');
67});
68
69test('anything else goes through oEmbed discovery', async () => {
70 const r = await resolveEmbed('https://v.example/watch/1', io({
71 getPage: async () => OEMBED_PAGE,
72 getJSON: async () => OEMBED_JSON,
73 }));
74 assert.equal(r.kind, 'oembed');
75 assert.equal(r.title, 'A talk');
76 assert.equal(r.author.name, 'Ada');
77 assert.equal(r.provider, 'Vid');
78 assert.equal(r.media[0].url, 'https://v.example/t.jpg');
79 assert.ok(r.html.startsWith('<iframe'));
80});
81
82test('no oEmbed link, or a dead endpoint, still yields a usable link card', async () => {
83 const noLink = await resolveEmbed('https://plain.example/p', io({ getPage: async () => '<html></html>' }));
84 assert.equal(noLink.kind, 'link');
85 const deadEndpoint = await resolveEmbed('https://v.example/p', io({
86 getPage: async () => OEMBED_PAGE, getJSON: async () => null,
87 }));
88 assert.equal(deadEndpoint.kind, 'link');
89});
90
91test('a failing AP fetch does not abort the pipeline', async () => {
92 const r = await resolveEmbed('https://v.example/p', io({
93 getAP: async () => { throw new Error('boom'); },
94 getPage: async () => OEMBED_PAGE,
95 getJSON: async () => OEMBED_JSON,
96 }));
97 assert.equal(r.kind, 'oembed');
98});
99
100test('non-http input is refused', async () => {
101 assert.equal(await resolveEmbed('javascript:alert(1)', io()), null);
102 assert.equal(await resolveEmbed('', io()), null);
103});
104
105test('fromOEmbed keeps only an http(s) canonical url', () => {
106 const c = fromOEmbed('https://a/b', { url: 'javascript:alert(1)', title: 'x' });
107 assert.equal(c.url, 'https://a/b');
108});
109
110// The live binding: the ordering above stays, but every fetch is capped and
111// goes through safeFetch, so a URL in a post cannot make us probe internals.
112test('liveIO caps oversized bodies and never throws on a bad fetch', async () => {
113 const { liveIO } = await import('../src/services/EmbedResolver.js');
114 const calls = [];
115 const fakeFetch = async (u, o) => {
116 calls.push([u, o.headers.Accept]);
117 if (u.includes('huge')) return { ok: true, headers: { get: () => String(9_000_000) }, text: async () => 'x' };
118 if (u.includes('boom')) throw new Error('refused');
119 return { ok: true, headers: { get: () => '10' }, text: async () => '{"type":"Note","id":"https://s/1"}' };
120 };
121 const io = liveIO({ safeFetch: fakeFetch, detectProvider: () => null });
122 // A JSON payload must parse whole, so an oversized one is refused outright.
123 assert.equal(await io.getJSON('https://x/huge'), null, 'oversized JSON refused');
124 assert.equal(await io.getAP('https://x/boom'), null, 'a refused fetch is not an error');
125 assert.deepEqual(await io.getAP('https://x/ok'), { type: 'Note', id: 'https://s/1' });
126 assert.ok(calls.some((c) => c[1].includes('activity+json')), 'AP asks for activity+json');
127});
128
129// OpenGraph: the one that actually carries link previews on the open web.
130// oEmbed is richer, but most sites simply do not ship it, which is why cards
131// stayed empty until this fallback existed.
132const OG_PAGE = '<html><head><meta property="og:title" content="Linux f&amp;uuml;r Einsteiger">'
133 + '<meta property="og:site_name" content="Linux Guides">'
134 + '<meta property="og:image" content="https://lg.example/tux.png"></head></html>';
135
136test('findOpenGraph reads og:image/title/site and decodes entities', async () => {
137 const { findOpenGraph } = await import('../src/services/EmbedResolver.js');
138 const og = findOpenGraph(OG_PAGE);
139 assert.equal(og.image, 'https://lg.example/tux.png');
140 assert.equal(og.site, 'Linux Guides');
141 assert.ok(og.title.startsWith('Linux f'));
142 assert.equal(findOpenGraph('<html><head><title>x</title></head></html>'), null);
143 assert.equal(findOpenGraph(null), null);
144});
145
146test('findOpenGraph falls back to twitter:image and refuses a non-http image', async () => {
147 const { findOpenGraph } = await import('../src/services/EmbedResolver.js');
148 const tw = findOpenGraph('<meta name="twitter:image" content="https://x/y.png"><meta name="twitter:title" content="T">');
149 assert.equal(tw.image, 'https://x/y.png');
150 const bad = findOpenGraph('<meta property="og:image" content="javascript:alert(1)"><meta property="og:title" content="T">');
151 assert.equal(bad.image, null, 'a non-http image is dropped, the title survives');
152 assert.equal(bad.title, 'T');
153});
154
155test('oEmbed still wins over OpenGraph when a page offers both', async () => {
156 const both = OEMBED_PAGE.replace('</head>', '<meta property="og:title" content="OG"></head>');
157 const r = await resolveEmbed('https://v.example/1', io({
158 getPage: async () => both, getJSON: async () => OEMBED_JSON,
159 }));
160 assert.equal(r.kind, 'oembed');
161 assert.equal(r.title, 'A talk');
162});
163
164test('a page with only OpenGraph yields a thumbnail card', async () => {
165 const r = await resolveEmbed('https://lg.example/artikel', io({ getPage: async () => OG_PAGE }));
166 assert.equal(r.kind, 'opengraph');
167 assert.equal(r.media[0].url, 'https://lg.example/tux.png');
168 assert.equal(r.provider, 'Linux Guides');
169 assert.equal(r.url, 'https://lg.example/artikel');
170});
171
172// Regression, the one that kept YouTube blank: a page is read from the START and
173// cut off, never refused for being large. Refusing it meant no thumbnail at all
174// for exactly the sites people share most.
175test('a huge page is truncated, not rejected', async () => {
176 const { liveIO } = await import('../src/services/EmbedResolver.js');
177 const big = '<html><head>' + 'x'.repeat(5000) + '<meta property="og:title" content="T">'
178 + '<meta property="og:image" content="https://x/i.png"></head></html>';
179 const fakeFetch = async () => ({ ok: true, headers: { get: () => String(9_000_000) }, text: async () => big });
180 const io = liveIO({ safeFetch: fakeFetch, detectProvider: () => null });
181 const page = await io.getPage('https://x/huge');
182 assert.ok(page && page.includes('og:image'), 'the head survives the cap');
183});
184
185// Regression: big sites carry the bare string "og:image" inside inline JSON long
186// before the real meta tag. Stopping the read there cut the page off just short
187// of the tags and produced no card at all.
188test('the head read does not stop on an og:image mention inside a script', async () => {
189 const { liveIO } = await import('../src/services/EmbedResolver.js');
190 const page = '<html><head><script>var cfg={"og:image":"decoy"};</script>'
191 + 'y'.repeat(3000)
192 + '<meta property="og:title" content="Echt"><meta property="og:image" content="https://x/real.png">'
193 + '</head></html>';
194 const fakeFetch = async () => ({ ok: true, headers: { get: () => '999' }, text: async () => page });
195 const io = liveIO({ safeFetch: fakeFetch, detectProvider: () => null });
196 const got = await io.getPage('https://x/p');
197 const { findOpenGraph } = await import('../src/services/EmbedResolver.js');
198 assert.equal(findOpenGraph(got).image, 'https://x/real.png', 'reads past the decoy to the real tag');
199});
200
201// The public oEmbed registry (oembed.com). Not a list we maintain: we read one
202// that is published. It exists because discovery through the page fails exactly
203// where it matters most, from a server YouTube hands a stripped page to.
204const PROVIDERS = [
205 { provider_name: 'YouTube', provider_url: 'https://www.youtube.com/',
206 endpoints: [{ schemes: ['https://*.youtube.com/watch*', 'https://youtu.be/*'], url: 'https://www.youtube.com/oembed' }] },
207 { provider_name: 'Bare', provider_url: 'https://bare.example/', endpoints: [{ url: 'https://bare.example/oembed.{format}' }] },
208];
209
210test('matchProviderEndpoint matches wildcard schemes and falls back to the host', async () => {
211 const { matchProviderEndpoint } = await import('../src/services/EmbedResolver.js');
212 assert.equal(matchProviderEndpoint('https://youtu.be/abc?is=x', PROVIDERS), 'https://www.youtube.com/oembed');
213 assert.equal(matchProviderEndpoint('https://www.youtube.com/watch?v=abc', PROVIDERS), 'https://www.youtube.com/oembed');
214 assert.equal(matchProviderEndpoint('https://bare.example/thing/1', PROVIDERS), 'https://bare.example/oembed.json',
215 'no schemes listed, so the provider host decides, and {format} is filled in');
216 assert.equal(matchProviderEndpoint('https://elders.example/x', PROVIDERS), null);
217 assert.equal(matchProviderEndpoint('not a url', PROVIDERS), null);
218});
219
220test('oembedRequestUrl appends url + format, keeping an existing query', async () => {
221 const { oembedRequestUrl } = await import('../src/services/EmbedResolver.js');
222 assert.ok(oembedRequestUrl('https://x/oembed', 'https://a/b?c=1').includes('format=json&url=https%3A%2F%2Fa%2Fb%3Fc%3D1'));
223 assert.ok(oembedRequestUrl('https://x/oembed?k=1', 'https://a/b').startsWith('https://x/oembed?k=1&'));
224});
225
226test('the registry is tried before the page, and the page is not fetched when it hits', async () => {
227 let pageFetched = false;
228 const r = await resolveEmbed('https://youtu.be/abc', io({
229 registry: async () => PROVIDERS,
230 getJSON: async () => OEMBED_JSON,
231 getPage: async () => { pageFetched = true; return OG_PAGE; },
232 }));
233 assert.equal(r.kind, 'oembed');
234 assert.equal(r.title, 'A talk');
235 assert.ok(!pageFetched, 'a registry hit saves the whole page download');
236});
237
238test('an unreachable registry just falls through to the page', async () => {
239 const r = await resolveEmbed('https://lg.example/a', io({
240 registry: async () => { throw new Error('offline'); },
241 getPage: async () => OG_PAGE,
242 }));
243 assert.equal(r.kind, 'opengraph');
244});
Note: See TracBrowser for help on using the repository browser.