Changeset 7d01696 in Klonkt

Timestamp:

07/30/2026 07:56:33 AM (6 weeks ago)

Author:

Robin <roboburr@…>

Branches:

main

Children:

9ed940e

Parents:

97bcf7e

Message:

Posterframes voor video, ffmpeg-optioneel

shaer-zowq, de serverhelft. Bij een video-upload trekt ffmpeg een frame op
1 seconde naar <naam>.poster.jpg, best-effort en buiten de responspad: op een
machine zonder ffmpeg gebeurt er niets en breekt er niets (de clients halen
dan zelf een frame, native). Vanaf daar reist de poster naar drie plekken:

• poster= op de gevouwen video-tag, dus het web toont een stilstaand beeld
• opgeslagen op de c2s_attachments-entry
• als AS2 icon op het gefedereerde Video-attachment, dus apps en andere servers krijgen de thumbnail-URL cadeau

Inkomend geldt het spiegelbeeld: een remote video-attachment met een icon
houdt hem als poster in media_json, en de C2S-inboxlees serveert hem terug.

Changed files:
src/routes/activitypub.js

• uploadMedia: ffmpeg-posterframe voor video, best-effort

src/services/ActivityPubService.js

• c2sCreatePost: poster op entry en tag; buildNote: icon op het attachment; mediaFromNote/timelineAttachments: inkomende posters door

test/c2s-compose.test.js

• de poster bereikt tag, opslag en attachment; zonder posterbestand wordt er niets gegokt

remarks: 338 tests groen. ffmpeg staat NIET op de VPS; installeren is een
sudo-actie die ik niet mag doen: sudo apt-get install -y ffmpeg. Zonder dat
werkt alles, alleen maken de clients hun thumbnails zelf.

-robo
Co-Authored-By: Claude Opus 5 <noreply@…>

Files:

• src/routes/activitypub.js (modified) (1 diff)
• src/services/ActivityPubService.js (modified) (7 diffs)
• test/c2s-compose.test.js (modified) (1 diff)

src/routes/activitypub.js

@@ -312 +312 @@
       return res.status(400).json({ error: 'Media must be an image, audio or video file' });
     }
+    // A video gets a poster frame next to it (shaer-zowq), best-effort and
+    // out of band: ffmpeg pulls one frame at 1s into <name>.poster.jpg. On a
+    // machine without ffmpeg nothing happens and nothing breaks; the clients
+    // fall back to extracting a frame natively.
+    if (mime.startsWith('video/')) {
+      import('child_process').then(({ execFile }) => {
+        const poster = req.file.path + '.poster.jpg';
+        execFile('ffmpeg', ['-y', '-ss', '1', '-i', req.file.path, '-frames:v', '1', '-vf', "scale='min(640,iw)':-2", poster],
+          { timeout: 30000 }, (e) => { if (e && e.code !== 'ENOENT') console.warn('[media] poster failed:', e.message); });
+      }).catch(() => { /* never blocks the upload */ });
+    }
     res.status(201).json({
       url: '/media/reply-media/' + req.file.filename,

src/services/ActivityPubService.js

@@ -17 +17 @@
  */
 import crypto from 'crypto';
+import fs from 'fs';
+import path from 'path';
 import dns from 'dns';
 import net from 'net';
@@ -380 +382 @@
   try {
     for (const a of JSON.parse(post.c2s_attachments || '[]')) {
-      if (a && a.url) urls.push({ url: abs(a.url), name: a.name || '', mt: a.mediaType });
+      if (a && a.url) urls.push({ url: abs(a.url), name: a.name || '', mt: a.mediaType, poster: a.poster ? abs(a.poster) : null });
     }
   } catch { /* malformed never blocks the Note */ }
@@ -480 +482 @@
       const a = { type: ty, mediaType: mt, url: x.url };
       if (x.name) a.name = String(x.name).slice(0, 1500); // alt text / description (AS2 `name`)
+      if (x.poster) a.icon = { type: 'Image', url: x.poster }; // the video's still (shaer-zowq)
       return a; });
   for (const a of openAudio) attachment.push(a); // fedi_open tracks → native Audio players
@@ -2379 +2382 @@
       && /^(image|audio|video)\//.test(String(a.mediaType || '')))
     .slice(0, 4)
-    .map((a) => ({ url: a.url, mediaType: String(a.mediaType), name: String(a.name || '').slice(0, 120) }));
+    .map((a) => {
+      const entry = { url: a.url, mediaType: String(a.mediaType), name: String(a.name || '').slice(0, 120) };
+      // A video's poster frame, when the upload leg made one (shaer-zowq):
+      // rides along so the tag, the federated attachment and the apps all
+      // show a still instead of a black box.
+      if (entry.mediaType.startsWith('video/')) {
+        try {
+          const mediaRoot = path.resolve(process.env.MEDIA_PATH || './storage/media');
+          const rel = entry.url.replace(/^\/media\//, '');
+          if (fs.existsSync(path.join(mediaRoot, rel + '.poster.jpg'))) entry.poster = entry.url + '.poster.jpg';
+        } catch { /* no poster is fine */ }
+      }
+      return entry;
+    });
   if (!html.trim() && !media.length) return { status: 400, error: 'empty_note' };
   // The web reads the post's content, so the media goes IN it (we build these
@@ -2389 +2405 @@
     if (a.mediaType.startsWith('image/')) return `<p><img src="${a.url}" alt="${esc(a.name)}"></p>`;
     if (a.mediaType.startsWith('audio/')) return `<p><audio controls preload="metadata" src="${a.url}"></audio></p>`;
-    return `<p><video controls playsinline preload="metadata" src="${a.url}"></video></p>`;
+    const poster = a.poster ? ` poster="${a.poster}"` : '';
+    return `<p><video controls playsinline preload="metadata"${poster} src="${a.url}"></video></p>`;
   }).join('');
   const postId = crypto.randomUUID();
@@ -2818 +2835 @@
     const rows = (Array.isArray(list) ? list : [])
       .filter((m) => m && m.url)
-      .map((m) => ({ type: 'Document', mediaType: m.type || undefined, url: m.url }));
+      .map((m) => {
+        const a = { type: 'Document', mediaType: m.type || undefined, url: m.url };
+        if (m.poster) a.icon = { type: 'Image', url: m.poster }; // the video's still (shaer-zowq)
+        return a;
+      });
     return rows.length ? rows : undefined;
   } catch { return undefined; }
@@ -3027 +3048 @@
 }
 function mediaFromNote(note) {
-  const atts = (Array.isArray(note.attachment) ? note.attachment : []).map((a) => ({ url: safeUrl(a && a.url), type: (a && a.mediaType) || '' })).filter((m) => m.url);
+  const atts = (Array.isArray(note.attachment) ? note.attachment : []).map((a) => {
+    const m = { url: safeUrl(a && a.url), type: (a && a.mediaType) || '' };
+    // A federated video may carry its poster as an AS2 icon (shaer-zowq).
+    const iconUrl = a && a.icon && safeUrl(typeof a.icon === 'string' ? a.icon : a.icon.url);
+    if (iconUrl && /^video\//i.test(m.type)) m.poster = iconUrl;
+    return m;
+  }).filter((m) => m.url);
   if (!atts.some((m) => !m.type || /image/i.test(m.type)) && note.image) {
     const im = Array.isArray(note.image) ? note.image[0] : note.image;

test/c2s-compose.test.js

@@ -55 +55 @@
 });
 
+test("a video's poster frame rides the tag, the store and the federated attachment", async () => {
+  // The upload leg writes <name>.poster.jpg next to a video when ffmpeg is
+  // around (shaer-zowq). From there it must reach three places: the poster=
+  // on the folded tag (web), the stored entry, and the AS2 icon on the
+  // federated attachment (apps and other servers).
+  const os = await import('os');
+  const fsm = await import('fs');
+  const pathm = await import('path');
+  const root = fsm.mkdtempSync(pathm.join(os.tmpdir(), 'klonkt-media-'));
+  fsm.mkdirSync(pathm.join(root, 'reply-media'), { recursive: true });
+  fsm.writeFileSync(pathm.join(root, 'reply-media', 'film.mp4.poster.jpg'), 'x');
+  const prev = process.env.MEDIA_PATH;
+  process.env.MEDIA_PATH = root;
+  try {
+    const r = await AP.ingestOutboxActivity(site, user, {
+      type: 'Create',
+      object: {
+        type: 'Note', content: '<p>filmpje</p>',
+        to: ['https://test.example/ap/users/kid/followers'],
+        attachment: [{ type: 'Video', url: '/media/reply-media/film.mp4', mediaType: 'video/mp4' }],
+      },
+    });
+    assert.equal(r.status, 201);
+    const post = db.prepare('SELECT * FROM posts WHERE id = ?').get(r.id);
+    assert.match(post.content, /poster="\/media\/reply-media\/film\.mp4\.poster\.jpg"/, 'the web tag shows the still');
+    const note = AP.buildNote('https://test.example', site, post);
+    const vid = (note.attachment || []).find((a) => a.url.endsWith('film.mp4'));
+    assert.ok(vid && vid.type === 'Video');
+    assert.equal(vid.icon && vid.icon.url, 'https://test.example/media/reply-media/film.mp4.poster.jpg',
+      'the poster federates as the attachment icon');
+  } finally {
+    if (prev === undefined) delete process.env.MEDIA_PATH; else process.env.MEDIA_PATH = prev;
+  }
+});
+
+test('a video without a poster simply has none: no guessed icon', async () => {
+  const r = await AP.ingestOutboxActivity(site, user, {
+    type: 'Create',
+    object: {
+      type: 'Note', content: '<p>kaal</p>',
+      to: ['https://test.example/ap/users/kid/followers'],
+      attachment: [{ type: 'Video', url: '/media/reply-media/zonder.mp4', mediaType: 'video/mp4' }],
+    },
+  });
+  const post = db.prepare('SELECT * FROM posts WHERE id = ?').get(r.id);
+  assert.ok(!post.content.includes('poster='), 'no poster attr without a poster file');
+  const vid = (AP.buildNote('https://test.example', site, post).attachment || []).find((a) => a.url.endsWith('zonder.mp4'));
+  assert.equal(vid.icon, undefined);
+});
+
 test('a media-only post is a post, not an empty-note error', async () => {
   const r = await AP.ingestOutboxActivity(site, user, {