Changeset bdb78e4 in Klonkt

Timestamp:

07/28/2026 08:59:00 AM (6 weeks ago)

Author:

Robin Genis <roboburr@…>

Branches:

main

Children:

2a76184

Parents:

b1512e0

Message:

Publiek oEmbed-register erbij: YouTube verstopt zijn tags voor een server

Waarom het op mijn machine werkte en op de VPS niet: YouTube serveert een
datacenter-IP een uitgeklede pagina. Titel leeg, nul meta property-tags, geen
oEmbed-link. Hun oEmbed-API antwoordt daar wel gewoon (status 200, echte titel).
Discovery via de pagina faalt dus precies waar het het meest uitmaakt.

Het publieke providerregister van oembed.com dicht dat gat, en dat is iets anders
dan de whitelist die we net weggehaald hebben: die lijst onderhouden wij niet, we
lezen er een die gepubliceerd wordt. Valt oembed.com weg, dan vallen we terug op
paginadiscovery en werkt alles gewoon minder slim.

Het register komt VOOR de paginafetch: het is een match in het geheugen plus een
klein API-verzoek, en scheelt dus ook de download van een pagina van een MB.
Eenmaal opgehaald blijft de lijst een dag staan; een stale lijst is beter dan
geen lijst.

Changed files:
src/services/EmbedResolver.js

• matchProviderEndpoint (wildcard-schemes, host-fallback, {format} invullen)
• oembedRequestUrl; registry-stap voor de pagina; liveIO cachet de lijst 24u

src/services/ActivityPubService.js

• SELFHEAL_VERSION 19 -> 20

test/embed-resolver.test.js

• 4 tests: schemes en host-fallback, url-opbouw, register wint van de pagina (en bespaart de download), en een onbereikbaar register valt netjes terug

remarks: 219 tests groen.

-robo
Co-Authored-By: Claude Opus 4.8 <noreply@…>

Files:

• src/services/ActivityPubService.js (modified) (1 diff)
• src/services/EmbedResolver.js (modified) (4 diffs)
• test/embed-resolver.test.js (modified) (1 diff)

src/services/ActivityPubService.js

@@ -2849 +2849 @@
 // during a flux window, e.g. a fleet-wide update), and drops notes that are gone
 // (404/410). Bump SELFHEAL_VERSION only on a release that warrants a re-sync.
-const SELFHEAL_VERSION = 19; // v19: head-read no longer stops on an og:image mention inside inline script
+const SELFHEAL_VERSION = 20; // v20: oEmbed provider registry, so platforms that hide their tags from a server still resolve
 async function fetchNoteAP(url) {
   try {

src/services/EmbedResolver.js

@@ -59 +59 @@
   if (!image && !title) return null;
   return { image: image && /^https?:\/\//i.test(image) ? image : null, title: title || null, site: meta['og:site_name'] || null };
+}
+
+/**
+ * Match a URL against the public oEmbed provider registry (oembed.com).
+ *
+ * Discovery through the page is the pure way, but it only works when the page
+ * hands you its <link rel=oembed>, and big platforms do not always do that: from
+ * a datacentre IP, YouTube serves a stripped page with no oEmbed link and no
+ * OpenGraph at all, while its oEmbed API answers perfectly. The registry closes
+ * that gap without us keeping a list of hosts: it is published and maintained by
+ * oembed.com, we only read it.
+ *
+ * Pure, so the pattern matching is testable without a network.
+ */
+export function matchProviderEndpoint(url, providers) {
+  if (!Array.isArray(providers)) return null;
+  let host = '';
+  try { host = new URL(url).host.replace(/^www\./, ''); } catch { return null; }
+  const toRe = (scheme) => new RegExp('^' + String(scheme)
+    .replace(/[.+?^${}()|[\]\\]/g, '\\$&')
+    .replace(/\*/g, '.*') + '$', 'i');
+  for (const p of providers) {
+    for (const ep of (p.endpoints || [])) {
+      const target = typeof ep.url === 'string' ? ep.url.replace('{format}', 'json') : null;
+      if (!target) continue;
+      for (const scheme of (ep.schemes || [])) {
+        if (toRe(scheme).test(url)) return target;
+      }
+      // No schemes listed: fall back to the provider's own host.
+      if (!ep.schemes || !ep.schemes.length) {
+        let phost = '';
+        try { phost = new URL(p.provider_url).host.replace(/^www\./, ''); } catch { /* skip */ }
+        if (phost && (host === phost || host.endsWith('.' + phost))) return target;
+      }
+    }
+  }
+  return null;
+}
+
+/** Add the url + json format to an oEmbed endpoint. */
+export function oembedRequestUrl(endpoint, url) {
+  const sep = endpoint.includes('?') ? '&' : '?';
+  return `${endpoint}${sep}format=json&url=${encodeURIComponent(url)}`;
 }
 
@@ -136 +179 @@
   }
 
-  // 2. oEmbed, then OpenGraph. One page fetch serves both: oEmbed is the richer
-  //    protocol, OpenGraph is the one most of the web actually ships.
+  // 2. The oEmbed registry: a cheap in-memory match, then one small API call.
+  //    Tried before the page because it is far cheaper AND because the big
+  //    platforms are exactly the ones that hide their tags from a server.
+  if (io.registry && io.getJSON) {
+    const providers = await io.registry().catch(() => null);
+    const endpoint = matchProviderEndpoint(url, providers);
+    if (endpoint) {
+      const o = await io.getJSON(oembedRequestUrl(endpoint, url)).catch(() => null);
+      const card = fromOEmbed(url, o);
+      if (card) return card;
+    }
+  }
+
+  // 3. oEmbed via the page, then OpenGraph. One page fetch serves both: oEmbed
+  //    is the richer protocol, OpenGraph is the one most of the web ships.
   //
   //    There is deliberately NO list of known providers here. A hardcoded list
@@ -247 +303 @@
  * @param {object} deps - { safeFetch, detectProvider, actorInfo, fetchActor }
  */
+// The provider registry, fetched once and kept for a day. It is a public list
+// maintained by oembed.com, not by us; if it is unreachable we simply fall back
+// to page discovery, so nothing breaks, it just gets less clever.
+const REGISTRY_URL = 'https://oembed.com/providers.json';
+const REGISTRY_TTL = 24 * 60 * 60 * 1000;
+let _registry = null;
+let _registryAt = 0;
+
 export function liveIO({ safeFetch, detectProvider, fetchActor, actorInfo }) {
   return {
+    registry: async () => {
+      if (_registry && Date.now() - _registryAt < REGISTRY_TTL) return _registry;
+      const body = await safeJsonText(safeFetch, REGISTRY_URL, 'application/json');
+      if (!body) return _registry;                       // keep a stale list over none
+      try { _registry = JSON.parse(body); _registryAt = Date.now(); } catch { /* keep the old one */ }
+      return _registry;
+    },
     provider: detectProvider ? (u) => { try { return detectProvider(u); } catch { return null; } } : null,
     getAP: async (u) => {
@@ -273 +344 @@
 }
 
-export default { resolveEmbed, findOEmbedEndpoint, findOpenGraph, looksLikeAPObject, fromOEmbed, fromAPObject, liveIO };
+export default { resolveEmbed, findOEmbedEndpoint, findOpenGraph, matchProviderEndpoint, oembedRequestUrl, looksLikeAPObject, fromOEmbed, fromAPObject, liveIO };

test/embed-resolver.test.js

@@ -198 +198 @@
   assert.equal(findOpenGraph(got).image, 'https://x/real.png', 'reads past the decoy to the real tag');
 });
+
+// The public oEmbed registry (oembed.com). Not a list we maintain: we read one
+// that is published. It exists because discovery through the page fails exactly
+// where it matters most, from a server YouTube hands a stripped page to.
+const PROVIDERS = [
+  { provider_name: 'YouTube', provider_url: 'https://www.youtube.com/',
+    endpoints: [{ schemes: ['https://*.youtube.com/watch*', 'https://youtu.be/*'], url: 'https://www.youtube.com/oembed' }] },
+  { provider_name: 'Bare', provider_url: 'https://bare.example/', endpoints: [{ url: 'https://bare.example/oembed.{format}' }] },
+];
+
+test('matchProviderEndpoint matches wildcard schemes and falls back to the host', async () => {
+  const { matchProviderEndpoint } = await import('../src/services/EmbedResolver.js');
+  assert.equal(matchProviderEndpoint('https://youtu.be/abc?is=x', PROVIDERS), 'https://www.youtube.com/oembed');
+  assert.equal(matchProviderEndpoint('https://www.youtube.com/watch?v=abc', PROVIDERS), 'https://www.youtube.com/oembed');
+  assert.equal(matchProviderEndpoint('https://bare.example/thing/1', PROVIDERS), 'https://bare.example/oembed.json',
+    'no schemes listed, so the provider host decides, and {format} is filled in');
+  assert.equal(matchProviderEndpoint('https://elders.example/x', PROVIDERS), null);
+  assert.equal(matchProviderEndpoint('not a url', PROVIDERS), null);
+});
+
+test('oembedRequestUrl appends url + format, keeping an existing query', async () => {
+  const { oembedRequestUrl } = await import('../src/services/EmbedResolver.js');
+  assert.ok(oembedRequestUrl('https://x/oembed', 'https://a/b?c=1').includes('format=json&url=https%3A%2F%2Fa%2Fb%3Fc%3D1'));
+  assert.ok(oembedRequestUrl('https://x/oembed?k=1', 'https://a/b').startsWith('https://x/oembed?k=1&'));
+});
+
+test('the registry is tried before the page, and the page is not fetched when it hits', async () => {
+  let pageFetched = false;
+  const r = await resolveEmbed('https://youtu.be/abc', io({
+    registry: async () => PROVIDERS,
+    getJSON: async () => OEMBED_JSON,
+    getPage: async () => { pageFetched = true; return OG_PAGE; },
+  }));
+  assert.equal(r.kind, 'oembed');
+  assert.equal(r.title, 'A talk');
+  assert.ok(!pageFetched, 'a registry hit saves the whole page download');
+});
+
+test('an unreachable registry just falls through to the page', async () => {
+  const r = await resolveEmbed('https://lg.example/a', io({
+    registry: async () => { throw new Error('offline'); },
+    getPage: async () => OG_PAGE,
+  }));
+  assert.equal(r.kind, 'opengraph');
+});