Render MSC2530 captions for images/videos in the timeline

Note: Requires changes in rust-sdk (and rust-components) and ruma. Change-Id: I3c239474aeacc408904ab23b1a8030a76e3c878d
SchildiChat · Dec 30, 2023 · e65d7cb · e65d7cb
1 parent ddf63f4
commit e65d7cb
Show file tree

Hide file tree

Showing 19 changed files with 108 additions and 16 deletions.
diff --git a/...lin/io/element/android/features/messages/impl/timeline/components/TimelineItemEventRow.kt b/...lin/io/element/android/features/messages/impl/timeline/components/TimelineItemEventRow.kt
@@ -86,6 +86,7 @@ import io.element.android.features.messages.impl.timeline.model.event.TimelineIt
 import io.element.android.features.messages.impl.timeline.model.event.TimelineItemVideoContent
 import io.element.android.features.messages.impl.timeline.model.event.aTimelineItemImageContent
 import io.element.android.features.messages.impl.timeline.model.event.aTimelineItemTextContent
+import io.element.android.features.messages.impl.timeline.model.event.timestampPosition
 import io.element.android.features.messages.impl.timeline.model.metadata
 import io.element.android.libraries.androidutils.system.openUrlInExternalApp
 import io.element.android.libraries.designsystem.colors.AvatarColorsProvider
@@ -520,7 +521,7 @@ private fun MessageEventBubbleContent(
             inReplyToDetails != null -> {
                 if (timestampPosition == TimestampPosition.Overlay) {
                     timestampLayoutModifier = Modifier.padding(start = 8.dp, end = 8.dp, bottom = 8.dp)
-                    contentModifier = Modifier.clip(RoundedCornerShape(12.dp))
+                    contentModifier = Modifier.clip(RoundedCornerShape(ScTheme.exposures.bubbleRadius))
                 } else {
                     contentModifier = Modifier.padding(start = 12.dp, end = 12.dp, top = 0.dp, bottom = 8.dp)
                     timestampLayoutModifier = Modifier
@@ -596,8 +597,8 @@ private fun MessageEventBubbleContent(
     }
 
     val timestampPosition = when (event.content) {
-        is TimelineItemImageContent,
-        is TimelineItemVideoContent,
+        is TimelineItemImageContent -> event.content.timestampPosition()
+        is TimelineItemVideoContent -> event.content.timestampPosition()
         is TimelineItemLocationContent -> TimestampPosition.Overlay
         is TimelineItemPollContent -> TimestampPosition.Below
         else -> TimestampPosition.Default

diff --git a/...ndroid/features/messages/impl/timeline/components/TimelineItemEventRowWithReplyPreview.kt b/...ndroid/features/messages/impl/timeline/components/TimelineItemEventRowWithReplyPreview.kt
@@ -95,23 +95,23 @@ class InReplyToDetailsProvider : PreviewParameterProvider<InReplyToDetails> {
             ),
             aMessageContent(
                 body = "Video",
-                type = VideoMessageType("Video", MediaSource("url"), null),
+                type = VideoMessageType("Video", null, null, MediaSource("url"), null),
             ),
             aMessageContent(
                 body = "Audio",
-                type = AudioMessageType("Audio", MediaSource("url"), null),
+                type = AudioMessageType("Audio", null, null, MediaSource("url"), null),
             ),
             aMessageContent(
                 body = "Voice",
-                type = VoiceMessageType("Voice", MediaSource("url"), null, null),
+                type = VoiceMessageType("Voice", null, null, MediaSource("url"), null, null),
             ),
             aMessageContent(
                 body = "Image",
-                type = ImageMessageType("Image", MediaSource("url"), null),
+                type = ImageMessageType("Image", null, null, MediaSource("url"), null),
             ),
             aMessageContent(
                 body = "File",
-                type = FileMessageType("File", MediaSource("url"), null),
+                type = FileMessageType("File", null, null, MediaSource("url"), null),
             ),
             aMessageContent(
                 body = "Location",

diff --git a/...n/io/element/android/features/messages/impl/timeline/components/event/ScCaptionWrapper.kt b/...n/io/element/android/features/messages/impl/timeline/components/event/ScCaptionWrapper.kt
@@ -0,0 +1,41 @@
+package io.element.android.features.messages.impl.timeline.components.event
+
+import androidx.compose.foundation.layout.Column
+import androidx.compose.foundation.layout.padding
+import androidx.compose.foundation.shape.RoundedCornerShape
+import androidx.compose.runtime.Composable
+import androidx.compose.ui.Modifier
+import androidx.compose.ui.draw.clip
+import androidx.compose.ui.unit.dp
+import chat.schildi.theme.ScTheme
+import io.element.android.features.messages.impl.timeline.model.event.TimelineItemTextContent
+
+@Composable
+fun ScCaptionWrapper(
+    caption: String?,
+    isEdited: Boolean,
+    extraPadding: ExtraPadding,
+    onLinkClicked: (url: String) -> Unit,
+    modifier: Modifier = Modifier,
+    content: @Composable (Modifier) -> Unit,
+) {
+    if (caption == null) {
+        content(modifier)
+    } else {
+        Column(Modifier) {
+            content(Modifier)
+            TimelineItemTextView(
+                content = TimelineItemTextContent(
+                    body = caption,
+                    htmlDocument = null,
+                    plainText = caption,
+                    formattedBody = null,
+                    isEdited = isEdited,
+                ),
+                extraPadding = extraPadding,
+                onLinkClicked = onLinkClicked,
+                modifier = Modifier.padding(start = 12.dp, end = 12.dp, top = 8.dp, bottom = 8.dp),
+            )
+        }
+    }
+}
diff --git a/.../android/features/messages/impl/timeline/components/event/TimelineItemEventContentView.kt b/.../android/features/messages/impl/timeline/components/event/TimelineItemEventContentView.kt
@@ -34,6 +34,8 @@ import io.element.android.features.messages.impl.timeline.model.event.TimelineIt
 import io.element.android.features.messages.impl.timeline.model.event.TimelineItemUnknownContent
 import io.element.android.features.messages.impl.timeline.model.event.TimelineItemVideoContent
 import io.element.android.features.messages.impl.timeline.model.event.TimelineItemVoiceContent
+import io.element.android.features.messages.impl.timeline.model.event.caption
+import io.element.android.features.messages.impl.timeline.model.event.isEdited
 import io.element.android.features.messages.impl.voicemessages.timeline.VoiceMessageState
 import io.element.android.libraries.architecture.Presenter
 
@@ -72,14 +74,14 @@ fun TimelineItemEventContentView(
             content = content,
             modifier = modifier
         )
-        is TimelineItemImageContent -> TimelineItemImageView(
+        is TimelineItemImageContent -> ScCaptionWrapper(content.caption(), content.isEdited(), extraPadding, onLinkClicked, modifier) { modifier -> TimelineItemImageView(
             content = content,
             modifier = modifier,
-        )
-        is TimelineItemVideoContent -> TimelineItemVideoView(
+        )}
+        is TimelineItemVideoContent -> ScCaptionWrapper(content.caption(), content.isEdited(), extraPadding, onLinkClicked, modifier) { modifier -> TimelineItemVideoView(
             content = content,
             modifier = modifier
-        )
+        )}
         is TimelineItemFileContent -> TimelineItemFileView(
             content = content,
             extraPadding = extraPadding,

diff --git a/...roid/features/messages/impl/timeline/factories/event/TimelineItemContentMessageFactory.kt b/...roid/features/messages/impl/timeline/factories/event/TimelineItemContentMessageFactory.kt
@@ -82,6 +82,7 @@ class TimelineItemContentMessageFactory @Inject constructor(
                 val aspectRatio = aspectRatioOf(messageType.info?.width, messageType.info?.height)
                 TimelineItemImageContent(
                     body = messageType.body,
+                    filename = messageType.filename,
                     mediaSource = messageType.source,
                     thumbnailSource = messageType.info?.thumbnailSource,
                     mimeType = messageType.info?.mimetype ?: MimeTypes.OctetStream,
@@ -115,6 +116,7 @@ class TimelineItemContentMessageFactory @Inject constructor(
                 val aspectRatio = aspectRatioOf(messageType.info?.width, messageType.info?.height)
                 TimelineItemVideoContent(
                     body = messageType.body,
+                    filename = messageType.filename,
                     thumbnailSource = messageType.info?.thumbnailSource,
                     videoSource = messageType.source,
                     mimeType = messageType.info?.mimetype ?: MimeTypes.OctetStream,
@@ -130,6 +132,7 @@ class TimelineItemContentMessageFactory @Inject constructor(
             is AudioMessageType -> {
                 TimelineItemAudioContent(
                     body = messageType.body,
+                    filename = messageType.filename,
                     mediaSource = messageType.source,
                     duration = messageType.info?.duration ?: Duration.ZERO,
                     mimeType = messageType.info?.mimetype ?: MimeTypes.OctetStream,
@@ -143,6 +146,7 @@ class TimelineItemContentMessageFactory @Inject constructor(
                         TimelineItemVoiceContent(
                             eventId = eventId,
                             body = messageType.body,
+                            filename = messageType.filename,
                             mediaSource = messageType.source,
                             duration = messageType.info?.duration ?: Duration.ZERO,
                             mimeType = messageType.info?.mimetype ?: MimeTypes.OctetStream,
@@ -152,6 +156,7 @@ class TimelineItemContentMessageFactory @Inject constructor(
                     false -> {
                         TimelineItemAudioContent(
                             body = messageType.body,
+                            filename = messageType.filename,
                             mediaSource = messageType.source,
                             duration = messageType.info?.duration ?: Duration.ZERO,
                             mimeType = messageType.info?.mimetype ?: MimeTypes.OctetStream,
@@ -165,6 +170,7 @@ class TimelineItemContentMessageFactory @Inject constructor(
                 val fileExtension = fileExtensionExtractor.extractFromName(messageType.body)
                 TimelineItemFileContent(
                     body = messageType.body,
+                    filename = messageType.filename,
                     thumbnailSource = messageType.info?.thumbnailSource,
                     fileSource = messageType.source,
                     mimeType = messageType.info?.mimetype ?: MimeTypes.fromFileExtension(fileExtension),

diff --git a/...lement/android/features/messages/impl/timeline/model/event/ScTimelineContentExtensions.kt b/...lement/android/features/messages/impl/timeline/model/event/ScTimelineContentExtensions.kt
@@ -0,0 +1,18 @@
+package io.element.android.features.messages.impl.timeline.model.event
+
+import io.element.android.features.messages.impl.timeline.components.TimestampPosition
+
+// MSC2530 helpers, not in original files to reduce upstream merge conflicts
+fun TimelineItemAudioContent.filenameOrBody() = filename ?: body
+fun TimelineItemAudioContent.caption() = body.takeIf { filename != null && filename != it }
+fun TimelineItemFileContent.filenameOrBody() = filename ?: body
+fun TimelineItemFileContent.caption() = body.takeIf { filename != null && filename != it }
+fun TimelineItemImageContent.filenameOrBody() = filename ?: body
+fun TimelineItemImageContent.caption() = body.takeIf { filename != null && filename != it }
+fun TimelineItemVideoContent.filenameOrBody() = filename ?: body
+fun TimelineItemVideoContent.caption() = body.takeIf { filename != null && filename != it }
+fun TimelineItemVoiceContent.filenameOrBody() = filename ?: body
+fun TimelineItemVoiceContent.caption() = body.takeIf { filename != null && filename != it }
+
+fun TimelineItemImageContent.timestampPosition() = if (caption() == null) TimestampPosition.Overlay else TimestampPosition.Aligned
+fun TimelineItemVideoContent.timestampPosition() = if (caption() == null) TimestampPosition.Overlay else TimestampPosition.Aligned
diff --git a/...o/element/android/features/messages/impl/timeline/model/event/TimelineItemAudioContent.kt b/...o/element/android/features/messages/impl/timeline/model/event/TimelineItemAudioContent.kt
@@ -21,6 +21,7 @@ import kotlin.time.Duration
 
 data class TimelineItemAudioContent(
     val body: String,
+    val filename: String?,
     val duration: Duration,
     val mediaSource: MediaSource,
     val mimeType: String,

diff --git a/...t/android/features/messages/impl/timeline/model/event/TimelineItemAudioContentProvider.kt b/...t/android/features/messages/impl/timeline/model/event/TimelineItemAudioContentProvider.kt
@@ -32,6 +32,7 @@ open class TimelineItemAudioContentProvider : PreviewParameterProvider<TimelineI
 
 fun aTimelineItemAudioContent(fileName: String = "A sound.mp3") = TimelineItemAudioContent(
     body = fileName,
+    filename = fileName,
     mimeType = MimeTypes.Pdf,
     formattedFileSize = "100kB",
     fileExtension = "mp3",

diff --git a/...io/element/android/features/messages/impl/timeline/model/event/TimelineItemFileContent.kt b/...io/element/android/features/messages/impl/timeline/model/event/TimelineItemFileContent.kt
@@ -21,6 +21,7 @@ import io.element.android.libraries.mediaviewer.api.helper.formatFileExtensionAn
 
 data class TimelineItemFileContent(
     val body: String,
+    val filename: String?,
     val fileSource: MediaSource,
     val thumbnailSource: MediaSource?,
     val formattedFileSize: String,

diff --git a/...nt/android/features/messages/impl/timeline/model/event/TimelineItemFileContentProvider.kt b/...nt/android/features/messages/impl/timeline/model/event/TimelineItemFileContentProvider.kt
@@ -31,6 +31,7 @@ open class TimelineItemFileContentProvider : PreviewParameterProvider<TimelineIt
 
 fun aTimelineItemFileContent(fileName: String = "A file.pdf") = TimelineItemFileContent(
     body = fileName,
+    filename = fileName,
     thumbnailSource = null,
     fileSource = MediaSource(url = ""),
     mimeType = MimeTypes.Pdf,

diff --git a/...o/element/android/features/messages/impl/timeline/model/event/TimelineItemImageContent.kt b/...o/element/android/features/messages/impl/timeline/model/event/TimelineItemImageContent.kt
@@ -21,6 +21,7 @@ import io.element.android.libraries.matrix.api.media.MediaSource
 
 data class TimelineItemImageContent(
     val body: String,
+    val filename: String?,
     val mediaSource: MediaSource,
     val thumbnailSource: MediaSource?,
     val formattedFileSize: String,

diff --git a/...t/android/features/messages/impl/timeline/model/event/TimelineItemImageContentProvider.kt b/...t/android/features/messages/impl/timeline/model/event/TimelineItemImageContentProvider.kt
@@ -32,6 +32,7 @@ open class TimelineItemImageContentProvider : PreviewParameterProvider<TimelineI
 
 fun aTimelineItemImageContent() = TimelineItemImageContent(
     body = "a body",
+    filename = null,
     mediaSource = MediaSource(""),
     thumbnailSource = null,
     mimeType = MimeTypes.IMAGE_JPEG,

diff --git a/...o/element/android/features/messages/impl/timeline/model/event/TimelineItemVideoContent.kt b/...o/element/android/features/messages/impl/timeline/model/event/TimelineItemVideoContent.kt
@@ -21,6 +21,7 @@ import kotlin.time.Duration
 
 data class TimelineItemVideoContent(
     val body: String,
+    val filename: String?,
     val duration: Duration,
     val videoSource: MediaSource,
     val thumbnailSource: MediaSource?,

diff --git a/...t/android/features/messages/impl/timeline/model/event/TimelineItemVideoContentProvider.kt b/...t/android/features/messages/impl/timeline/model/event/TimelineItemVideoContentProvider.kt
@@ -33,6 +33,7 @@ open class TimelineItemVideoContentProvider : PreviewParameterProvider<TimelineI
 
 fun aTimelineItemVideoContent() = TimelineItemVideoContent(
     body = "Video.mp4",
+    filename = null,
     thumbnailSource = null,
     blurHash = A_BLUR_HASH,
     aspectRatio = 0.5f,

diff --git a/...o/element/android/features/messages/impl/timeline/model/event/TimelineItemVoiceContent.kt b/...o/element/android/features/messages/impl/timeline/model/event/TimelineItemVoiceContent.kt
@@ -24,6 +24,7 @@ import kotlin.time.Duration
 data class TimelineItemVoiceContent(
     val eventId: EventId?,
     val body: String,
+    val filename: String?,
     val duration: Duration,
     val mediaSource: MediaSource,
     val mimeType: String,

diff --git a/...t/android/features/messages/impl/timeline/model/event/TimelineItemVoiceContentProvider.kt b/...t/android/features/messages/impl/timeline/model/event/TimelineItemVoiceContentProvider.kt
@@ -53,6 +53,7 @@ fun aTimelineItemVoiceContent(
 ) = TimelineItemVoiceContent(
     eventId = eventId?.let { EventId(it) },
     body = body,
+    filename = null,
     duration = duration,
     mediaSource = MediaSource(contentUri),
     mimeType = mimeType,

diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
@@ -150,7 +150,7 @@ jsoup = "org.jsoup:jsoup:1.17.1"
 appyx_core = { module = "com.bumble.appyx:core", version.ref = "appyx" }
 molecule-runtime = "app.cash.molecule:molecule-runtime:1.3.1"
 timber = "com.jakewharton.timber:timber:5.0.1"
-matrix_sdk = "chat.schildi.rustcomponents:sdk-android:0.1.69"
+matrix_sdk = "chat.schildi.rustcomponents:sdk-android:0.1.70"
 matrix_richtexteditor = { module = "io.element.android:wysiwyg", version.ref = "wysiwyg" }
 matrix_richtexteditor_compose = { module = "io.element.android:wysiwyg-compose", version.ref = "wysiwyg" }
 sqldelight-driver-android = { module = "app.cash.sqldelight:android-driver", version.ref = "sqldelight" }

diff --git a/...rc/main/kotlin/io/element/android/libraries/matrix/api/timeline/item/event/MessageType.kt b/...rc/main/kotlin/io/element/android/libraries/matrix/api/timeline/item/event/MessageType.kt
@@ -34,6 +34,8 @@ data class EmoteMessageType(
 
 data class ImageMessageType(
     val body: String,
+    val formatted: FormattedBody?,
+    val filename: String?,
     val source: MediaSource,
     val info: ImageInfo?
 ) : MessageType
@@ -46,25 +48,33 @@ data class LocationMessageType(
 
 data class AudioMessageType(
     val body: String,
+    val formatted: FormattedBody?,
+    val filename: String?,
     val source: MediaSource,
     val info: AudioInfo?,
 ) : MessageType
 
 data class VoiceMessageType(
     val body: String,
+    val formatted: FormattedBody?,
+    val filename: String?,
     val source: MediaSource,
     val info: AudioInfo?,
     val details: AudioDetails?,
 ) : MessageType
 
 data class VideoMessageType(
     val body: String,
+    val formatted: FormattedBody?,
+    val filename: String?,
     val source: MediaSource,
     val info: VideoInfo?
 ) : MessageType
 
 data class FileMessageType(
     val body: String,
+    val formatted: FormattedBody?,
+    val filename: String?,
     val source: MediaSource,
     val info: FileInfo?
 ) : MessageType

diff --git a/...kotlin/io/element/android/libraries/matrix/impl/timeline/item/event/EventMessageMapper.kt b/...kotlin/io/element/android/libraries/matrix/impl/timeline/item/event/EventMessageMapper.kt
@@ -81,13 +81,17 @@ class EventMessageMapper {
                 null -> {
                     AudioMessageType(
                         body = type.content.body,
+                        formatted = type.content.formatted?.map(),
+                        filename = type.content.filename,
                         source = type.content.source.map(),
                         info = type.content.info?.map(),
                     )
                 }
                 else -> {
                     VoiceMessageType(
                         body = type.content.body,
+                        formatted = type.content.formatted?.map(),
+                        filename = type.content.filename,
                         source = type.content.source.map(),
                         info = type.content.info?.map(),
                         details = type.content.audio?.map(),
@@ -96,10 +100,10 @@ class EventMessageMapper {
             }
         }
         is RustMessageType.File -> {
-            FileMessageType(type.content.body, type.content.source.map(), type.content.info?.map())
+            FileMessageType(type.content.body, type.content.formatted?.map(), type.content.filename, type.content.source.map(), type.content.info?.map())
         }
         is RustMessageType.Image -> {
-            ImageMessageType(type.content.body, type.content.source.map(), type.content.info?.map())
+            ImageMessageType(type.content.body, type.content.formatted?.map(), type.content.filename, type.content.source.map(), type.content.info?.map())
         }
         is RustMessageType.Notice -> {
             NoticeMessageType(type.content.body, type.content.formatted?.map())
@@ -111,7 +115,7 @@ class EventMessageMapper {
             EmoteMessageType(type.content.body, type.content.formatted?.map())
         }
         is RustMessageType.Video -> {
-            VideoMessageType(type.content.body, type.content.source.map(), type.content.info?.map())
+            VideoMessageType(type.content.body, type.content.formatted?.map(), type.content.filename, type.content.source.map(), type.content.info?.map())
         }
         is RustMessageType.Location -> {
             LocationMessageType(type.content.body, type.content.geoUri, type.content.description)