IntenseFinance/simplex-chat
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

desktop: enhanced video player + inline player (#3130)

Browse Source 
* desktop: enhanced video player + inline player

* simplify

* simplify

* removed unused code

* follow up

---------

Co-authored-by: Evgeny Poberezkin <2769109+epoberezkin@users.noreply.github.com>
This commit is contained in:
Stanislav Dmitrenko 2023-09-27 17:19:48 +08:00 committed by GitHub
parent 50d624ef6b
commit 8709ad6ff3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 283 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/multiplatform/common/build.gradle.kts
View File

@ -97,7 +97,7 @@ kotlin {
implementation("com.github.Dansoftowner:jSystemThemeDetector:3.6")
implementation("com.sshtools:two-slices:0.9.0-SNAPSHOT")
implementation("org.slf4j:slf4j-simple:2.0.7")
implementation("uk.co.caprica:vlcj:4.7.0")
implementation("uk.co.caprica:vlcj:4.7.3")
}
}
val desktopTest by getting

2
apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/chat/item/CIVIdeoView.kt
View File

@ -127,7 +127,7 @@ private fun VideoView(uri: URI, file: CIFile, defaultPreview: ImageBitmap, defau
)
if (showPreview.value) {
VideoPreviewImageView(preview, onClick, onLongClick)
PlayButton(brokenVideo, onLongClick = onLongClick, if (appPlatform.isAndroid) play else onClick)
PlayButton(brokenVideo, onLongClick = onLongClick, play)
}
DurationProgress(file, videoPlaying, duration, progress/*, soundEnabled*/)
}

5
apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/chat/item/ImageFullScreenView.kt
View File

@ -158,12 +158,11 @@ private fun VideoView(modifier: Modifier, uri: URI, defaultPreview: ImageBitmap,
player.stop()
}
LaunchedEffect(Unit) {
player.enableSound(true)
snapshotFlow { isCurrentPage.value }
.distinctUntilChanged()
.collect {
// Do not autoplay on desktop because it needs workaround
if (it && appPlatform.isAndroid) play() else if (!it) stop()
if (it) play() else stop()
player.enableSound(true)
}
}

79
apps/multiplatform/common/src/desktopMain/kotlin/chat/simplex/common/other/videoplayer/SkiaBitmapVideoSurface.kt Normal file
View File

@ -0,0 +1,79 @@
package org.jetbrains.compose.videoplayer
import androidx.compose.runtime.State
import androidx.compose.runtime.mutableStateOf
import androidx.compose.ui.graphics.ImageBitmap
import androidx.compose.ui.graphics.asComposeImageBitmap
import org.jetbrains.skia.Bitmap
import org.jetbrains.skia.ColorAlphaType
import org.jetbrains.skia.ColorType
import org.jetbrains.skia.ImageInfo
import uk.co.caprica.vlcj.player.base.MediaPlayer
import uk.co.caprica.vlcj.player.embedded.videosurface.CallbackVideoSurface
import uk.co.caprica.vlcj.player.embedded.videosurface.VideoSurface
import uk.co.caprica.vlcj.player.embedded.videosurface.VideoSurfaceAdapters
import uk.co.caprica.vlcj.player.embedded.videosurface.callback.BufferFormat
import uk.co.caprica.vlcj.player.embedded.videosurface.callback.BufferFormatCallback
import uk.co.caprica.vlcj.player.embedded.videosurface.callback.RenderCallback
import uk.co.caprica.vlcj.player.embedded.videosurface.callback.format.RV32BufferFormat
import java.nio.ByteBuffer
import javax.swing.SwingUtilities
// https://github.com/JetBrains/compose-multiplatform/pull/3336/files
internal class SkiaBitmapVideoSurface : VideoSurface(VideoSurfaceAdapters.getVideoSurfaceAdapter()) {
private val videoSurface = SkiaBitmapVideoSurface()
private lateinit var imageInfo: ImageInfo
private lateinit var frameBytes: ByteArray
private val skiaBitmap: Bitmap = Bitmap()
private val composeBitmap = mutableStateOf<ImageBitmap?>(null)
val bitmap: State<ImageBitmap?> = composeBitmap
override fun attach(mediaPlayer: MediaPlayer) {
videoSurface.attach(mediaPlayer)
}
private inner class SkiaBitmapBufferFormatCallback : BufferFormatCallback {
private var sourceWidth: Int = 0
private var sourceHeight: Int = 0
override fun getBufferFormat(sourceWidth: Int, sourceHeight: Int): BufferFormat {
this.sourceWidth = sourceWidth
this.sourceHeight = sourceHeight
return RV32BufferFormat(sourceWidth, sourceHeight)
}
override fun allocatedBuffers(buffers: Array<ByteBuffer>) {
frameBytes = buffers[0].run { ByteArray(remaining()).also(::get) }
imageInfo = ImageInfo(
sourceWidth,
sourceHeight,
ColorType.BGRA_8888,
ColorAlphaType.PREMUL,
)
}
}
private inner class SkiaBitmapRenderCallback : RenderCallback {
override fun display(
mediaPlayer: MediaPlayer,
nativeBuffers: Array<ByteBuffer>,
bufferFormat: BufferFormat,
) {
SwingUtilities.invokeLater {
nativeBuffers[0].rewind()
nativeBuffers[0].get(frameBytes)
skiaBitmap.installPixels(imageInfo, frameBytes, bufferFormat.width * 4)
composeBitmap.value = skiaBitmap.asComposeImageBitmap()
}
}
}
private inner class SkiaBitmapVideoSurface : CallbackVideoSurface(
SkiaBitmapBufferFormatCallback(),
SkiaBitmapRenderCallback(),
true,
videoSurfaceAdapter,
)
}

36
apps/multiplatform/common/src/desktopMain/kotlin/chat/simplex/common/platform/Images.desktop.kt
View File

@ -6,6 +6,8 @@ import boofcv.struct.image.GrayU8
import chat.simplex.res.MR
import org.jetbrains.skia.Image
import java.awt.RenderingHints
import java.awt.geom.AffineTransform
import java.awt.image.AffineTransformOp
import java.awt.image.BufferedImage
import java.io.*
import java.net.URI
@ -171,3 +173,37 @@ actual fun isAnimImage(uri: URI, drawable: Any?): Boolean {
@Suppress("NewApi")
actual fun loadImageBitmap(inputStream: InputStream): ImageBitmap =
Image.makeFromEncoded(inputStream.readAllBytes()).toComposeImageBitmap()
// https://stackoverflow.com/a/68926993
fun BufferedImage.rotate(angle: Double): BufferedImage {
val sin = Math.abs(Math.sin(Math.toRadians(angle)))
val cos = Math.abs(Math.cos(Math.toRadians(angle)))
val w = width
val h = height
val neww = Math.floor(w * cos + h * sin).toInt()
val newh = Math.floor(h * cos + w * sin).toInt()
val rotated = BufferedImage(neww, newh, type)
val graphic = rotated.createGraphics()
graphic.translate((neww - w) / 2, (newh - h) / 2)
graphic.rotate(Math.toRadians(angle), (w / 2).toDouble(), (h / 2).toDouble())
graphic.drawRenderedImage(this, null)
graphic.dispose()
return rotated
}
// https://stackoverflow.com/a/9559043
fun BufferedImage.flip(vertically: Boolean, horizontally: Boolean): BufferedImage {
if (!vertically && !horizontally) return this
val tx: AffineTransform
if (vertically && horizontally) {
tx = AffineTransform.getScaleInstance(-1.0, -1.0)
tx.translate(-width.toDouble(), -height.toDouble())
} else if (vertically) {
tx = AffineTransform.getScaleInstance(1.0, -1.0)
tx.translate(0.0, -height.toDouble())
} else {
tx = AffineTransform.getScaleInstance(-1.0, 1.0)
tx.translate(-width.toDouble(), 0.0)
}
return AffineTransformOp(tx, AffineTransformOp.TYPE_NEAREST_NEIGHBOR).filter(this, null)
}

179
apps/multiplatform/common/src/desktopMain/kotlin/chat/simplex/common/platform/VideoPlayer.desktop.kt
View File

@ -2,17 +2,20 @@ package chat.simplex.common.platform
import androidx.compose.runtime.MutableState
import androidx.compose.runtime.mutableStateOf
import androidx.compose.ui.graphics.ImageBitmap
import androidx.compose.ui.graphics.toComposeImageBitmap
import androidx.compose.ui.graphics.*
import chat.simplex.common.views.helpers.*
import chat.simplex.res.MR
import kotlinx.coroutines.*
import uk.co.caprica.vlcj.player.base.MediaPlayer
import uk.co.caprica.vlcj.media.VideoOrientation
import uk.co.caprica.vlcj.player.base.*
import uk.co.caprica.vlcj.player.component.CallbackMediaPlayerComponent
import uk.co.caprica.vlcj.player.component.EmbeddedMediaPlayerComponent
import java.awt.Component
import java.awt.image.BufferedImage
import java.io.File
import java.net.URI
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicBoolean
import kotlin.math.max
actual class VideoPlayer actual constructor(
@ -29,17 +32,14 @@ actual class VideoPlayer actual constructor(
override val duration: MutableState<Long> = mutableStateOf(0L)
override val preview: MutableState<ImageBitmap> = mutableStateOf(defaultPreview)
val mediaPlayerComponent = initializeMediaPlayerComponent()
val mediaPlayerComponent by lazy { runBlocking(playerThread.asCoroutineDispatcher()) { getOrCreatePlayer() } }
val player by lazy { mediaPlayerComponent.mediaPlayer() }
init {
withBGApi {
setPreviewAndDuration()
}
setPreviewAndDuration()
}
private val currentVolume: Int by lazy { player.audio().volume() }
private var isReleased: Boolean = false
private var isReleased: AtomicBoolean = AtomicBoolean(false)
private val listener: MutableState<((position: Long?, state: TrackState) -> Unit)?> = mutableStateOf(null)
private var progressJob: Job? = null
@ -48,6 +48,7 @@ actual class VideoPlayer actual constructor(
PLAYING, PAUSED, STOPPED
}
/** Should be called in [playerThread]. Otherwise, it creates deadlocks in [player.stop] and [player.release] calls */
private fun start(seek: Long? = null, onProgressUpdate: (position: Long?, state: TrackState) -> Unit): Boolean {
val filepath = getAppFilePath(uri)
if (filepath == null || !File(filepath).exists()) {
@ -87,7 +88,7 @@ actual class VideoPlayer actual constructor(
// Player can only be accessed in one specific thread
progressJob = CoroutineScope(Dispatchers.Main).launch {
onProgressUpdate(player.currentPosition.toLong(), TrackState.PLAYING)
while (isActive && !isReleased && player.isPlaying) {
while (isActive && !isReleased.get() && player.isPlaying) {
// Even when current position is equal to duration, the player has isPlaying == true for some time,
// so help to make the playback stopped in UI immediately
if (player.currentPosition == player.duration) {
@ -97,7 +98,7 @@ actual class VideoPlayer actual constructor(
delay(50)
onProgressUpdate(player.currentPosition.toLong(), TrackState.PLAYING)
}
if (isActive && !isReleased) {
if (isActive && !isReleased.get()) {
onProgressUpdate(player.currentPosition.toLong(), TrackState.PAUSED)
}
onProgressUpdate(null, TrackState.PAUSED)
@ -107,9 +108,11 @@ actual class VideoPlayer actual constructor(
}
override fun stop() {
if (isReleased || !videoPlaying.value) return
player.controls().stop()
stopListener()
if (isReleased.get() || !videoPlaying.value) return
playerThread.execute {
player.stop()
stopListener()
}
}
private fun stopListener() {
@ -133,45 +136,57 @@ actual class VideoPlayer actual constructor(
if (progress.value == duration.value) {
progress.value = 0
}
videoPlaying.value = start(progress.value) { pro, _ ->
if (pro != null) {
progress.value = pro
}
if ((pro == null || pro == duration.value) && duration.value != 0L) {
videoPlaying.value = false
if (pro == duration.value) {
progress.value = if (resetOnEnd) 0 else duration.value
}/* else if (state == TrackState.STOPPED) {
playerThread.execute {
videoPlaying.value = start(progress.value) { pro, _ ->
if (pro != null) {
progress.value = pro
}
if ((pro == null || pro == duration.value) && duration.value != 0L) {
videoPlaying.value = false
if (pro == duration.value) {
progress.value = if (resetOnEnd) 0 else duration.value
}/* else if (state == TrackState.STOPPED) {
progress.value = 0 //
}*/
}
}
}
}
override fun enableSound(enable: Boolean): Boolean {
if (isReleased) return false
if (soundEnabled.value == enable) return false
// Impossible to change volume for only one player. It changes for every player
// https://github.com/caprica/vlcj/issues/985
return false
/*if (isReleased.get() || soundEnabled.value == enable) return false
soundEnabled.value = enable
player.audio().setVolume(if (enable) currentVolume else 0)
return true
playerThread.execute {
player.audio().isMute = !enable
}
return true*/
}
override fun release(remove: Boolean) { withApi {
if (isReleased) return@withApi
isReleased = true
// TODO
/** [player.release] freezes thread for some reason. It happens periodically. So doing this we don't see the freeze, but it's still there */
if (player.isPlaying) player.stop()
CoroutineScope(Dispatchers.IO).launch { player.release() }
if (remove) {
VideoPlayerHolder.players.remove(uri to gallery)
override fun release(remove: Boolean) {
CoroutineScope(playerThread.asCoroutineDispatcher()).launch {
if (isReleased.get()) return@launch
isReleased.set(true)
if (player.isPlaying) {
player.stop()
}
if (usePool) {
putPlayer(mediaPlayerComponent)
} else {
player.release()
}
if (remove) {
VideoPlayerHolder.players.remove(uri to gallery)
}
}
}}
}
private val MediaPlayer.currentPosition: Int
get() = if (isReleased) 0 else max(0, player.status().time().toInt())
get() = if (isReleased.get()) 0 else max(0, status().time().toInt())
private suspend fun setPreviewAndDuration() {
private fun setPreviewAndDuration() {
// It freezes main thread, doing it in IO thread
CoroutineScope(Dispatchers.IO).launch {
val previewAndDuration = VideoPlayerHolder.previewsAndDurations.getOrPut(uri) { getBitmapFromVideo(defaultPreview, uri) }
@ -182,35 +197,79 @@ actual class VideoPlayer actual constructor(
}
}
private fun initializeMediaPlayerComponent(): Component {
return if (desktopPlatform.isMac()) {
CallbackMediaPlayerComponent()
} else {
EmbeddedMediaPlayerComponent()
}
}
private fun Component.mediaPlayer() = when (this) {
is CallbackMediaPlayerComponent -> mediaPlayer()
is EmbeddedMediaPlayerComponent -> mediaPlayer()
else -> error("mediaPlayer() can only be called on vlcj player components")
}
companion object {
suspend fun getBitmapFromVideo(defaultPreview: ImageBitmap?, uri: URI?): VideoPlayerInterface.PreviewAndDuration {
val player = CallbackMediaPlayerComponent().mediaPlayer()
private val usePool = false
private fun Component.mediaPlayer() = when (this) {
is CallbackMediaPlayerComponent -> mediaPlayer()
is EmbeddedMediaPlayerComponent -> mediaPlayer()
else -> error("mediaPlayer() can only be called on vlcj player components")
}
private fun initializeMediaPlayerComponent(): Component {
return if (desktopPlatform.isMac()) {
CallbackMediaPlayerComponent()
} else {
EmbeddedMediaPlayerComponent()
}
}
suspend fun getBitmapFromVideo(defaultPreview: ImageBitmap?, uri: URI?): VideoPlayerInterface.PreviewAndDuration = withContext(playerThread.asCoroutineDispatcher()) {
val mediaComponent = getOrCreateHelperPlayer()
val player = mediaComponent.mediaPlayer()
if (uri == null || !File(uri.rawPath).exists()) {
return VideoPlayerInterface.PreviewAndDuration(preview = defaultPreview, timestamp = 0L, duration = 0L)
return@withContext VideoPlayerInterface.PreviewAndDuration(preview = defaultPreview, timestamp = 0L, duration = 0L)
}
player.media().startPaused(uri.toString().replaceFirst("file:", "file://"))
val start = System.currentTimeMillis()
while (player.snapshots()?.get() == null && start + 5000 > System.currentTimeMillis()) {
var snap: BufferedImage? = null
while (snap == null && start + 5000 > System.currentTimeMillis()) {
snap = player.snapshots()?.get()
delay(10)
}
val preview = player.snapshots()?.get()?.toComposeImageBitmap()
val orientation = player.media().info().videoTracks().first().orientation()
val preview: ImageBitmap? = when (orientation) {
VideoOrientation.TOP_LEFT -> snap
VideoOrientation.TOP_RIGHT -> snap?.flip(false, true)
VideoOrientation.BOTTOM_LEFT -> snap?.flip(true, false)
VideoOrientation.BOTTOM_RIGHT -> snap?.rotate(180.0)
VideoOrientation.LEFT_TOP -> snap /* Transposed */
VideoOrientation.LEFT_BOTTOM -> snap?.rotate(-90.0)
VideoOrientation.RIGHT_TOP -> snap?.rotate(90.0)
VideoOrientation.RIGHT_BOTTOM -> snap /* Anti-transposed */
else -> snap
}?.toComposeImageBitmap()
val duration = player.duration.toLong()
CoroutineScope(Dispatchers.IO).launch { player.release() }
return VideoPlayerInterface.PreviewAndDuration(preview = preview, timestamp = 0L, duration = duration)
player.stop()
putHelperPlayer(mediaComponent)
return@withContext VideoPlayerInterface.PreviewAndDuration(preview = preview, timestamp = 0L, duration = duration)
}
val playerThread = Executors.newSingleThreadExecutor()
private val playersPool: ArrayList<Component> = ArrayList()
private val helperPlayersPool: ArrayList<CallbackMediaPlayerComponent> = ArrayList()
private fun getOrCreatePlayer(): Component = playersPool.removeFirstOrNull() ?: createNew()
private fun createNew(): Component =
initializeMediaPlayerComponent().apply {
mediaPlayer().events().addMediaPlayerEventListener(object: MediaPlayerEventAdapter() {
override fun mediaPlayerReady(mediaPlayer: MediaPlayer?) {
playerThread.execute {
mediaPlayer?.audio()?.setVolume(100)
mediaPlayer?.audio()?.isMute = false
}
}
override fun stopped(mediaPlayer: MediaPlayer?) {
//playerThread.execute { mediaPlayer().videoSurface().set(null) }
}
})
}
private fun putPlayer(player: Component) = playersPool.add(player)
private fun getOrCreateHelperPlayer(): CallbackMediaPlayerComponent = helperPlayersPool.removeFirstOrNull() ?: CallbackMediaPlayerComponent()
private fun putHelperPlayer(player: CallbackMediaPlayerComponent) = helperPlayersPool.add(player)
}
}

19
apps/multiplatform/common/src/desktopMain/kotlin/chat/simplex/common/views/chat/item/CIVideoView.desktop.kt
View File

@ -1,12 +1,29 @@
package chat.simplex.common.views.chat.item
import androidx.compose.foundation.combinedClickable
import androidx.compose.foundation.layout.*
import androidx.compose.runtime.*
import androidx.compose.ui.Modifier
import androidx.compose.ui.platform.LocalDensity
import androidx.compose.ui.unit.Dp
import chat.simplex.common.platform.VideoPlayer
import chat.simplex.common.platform.isPlaying
import chat.simplex.common.views.helpers.onRightClick
@Composable
actual fun PlayerView(player: VideoPlayer, width: Dp, onClick: () -> Unit, onLongClick: () -> Unit, stop: () -> Unit) {}
actual fun PlayerView(player: VideoPlayer, width: Dp, onClick: () -> Unit, onLongClick: () -> Unit, stop: () -> Unit) {
Box {
SurfaceFromPlayer(player,
Modifier
.width(width)
.combinedClickable(
onLongClick = onLongClick,
onClick = { if (player.player.isPlaying) stop() else onClick() }
)
.onRightClick(onLongClick)
)
}
}
@Composable
actual fun LocalWindowWidth(): Dp {

49
apps/multiplatform/common/src/desktopMain/kotlin/chat/simplex/common/views/chat/item/ImageFullScreenView.desktop.kt
View File

@ -6,17 +6,15 @@ import androidx.compose.material.*
import androidx.compose.runtime.*
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.awt.SwingPanel
import androidx.compose.ui.graphics.*
import androidx.compose.ui.layout.ContentScale
import androidx.compose.ui.unit.dp
import chat.simplex.common.platform.*
import chat.simplex.common.simplexWindowState
import chat.simplex.common.views.helpers.getBitmapFromByteArray
import chat.simplex.res.MR
import dev.icerock.moko.resources.compose.painterResource
import dev.icerock.moko.resources.compose.stringResource
import kotlinx.coroutines.delay
import org.jetbrains.compose.videoplayer.SkiaBitmapVideoSurface
import kotlin.math.max
@Composable
@ -28,30 +26,40 @@ actual fun FullScreenImageView(modifier: Modifier, data: ByteArray, imageBitmap:
modifier = modifier,
)
}
@Composable
actual fun FullScreenVideoView(player: VideoPlayer, modifier: Modifier, close: () -> Unit) {
// Workaround. Without changing size of the window the screen flashes a lot even if it's not being recomposed
LaunchedEffect(Unit) {
simplexWindowState.windowState.size = simplexWindowState.windowState.size.copy(width = simplexWindowState.windowState.size.width + 1.dp)
delay(50)
player.play(true)
simplexWindowState.windowState.size = simplexWindowState.windowState.size.copy(width = simplexWindowState.windowState.size.width - 1.dp)
}
Box {
Box(Modifier.fillMaxSize().padding(bottom = 50.dp)) {
val factory = remember { { player.mediaPlayerComponent } }
SwingPanel(
background = Color.Transparent,
modifier = Modifier,
factory = factory
)
SurfaceFromPlayer(player, modifier)
IconButton(onClick = close, Modifier.padding(top = 5.dp)) {
Icon(painterResource(MR.images.ic_arrow_back_ios_new), null, Modifier.size(30.dp), tint = MaterialTheme.colors.primary)
}
}
Controls(player, close)
Controls(player)
}
}
@Composable
private fun BoxScope.Controls(player: VideoPlayer, close: () -> Unit) {
fun BoxScope.SurfaceFromPlayer(player: VideoPlayer, modifier: Modifier) {
val surface = remember {
SkiaBitmapVideoSurface().also {
player.player.videoSurface().set(it)
}
}
surface.bitmap.value?.let { bitmap ->
Image(
bitmap,
modifier = modifier.align(Alignment.Center),
contentDescription = null,
contentScale = ContentScale.Fit,
alignment = Alignment.Center,
)
}
}
@Composable
private fun BoxScope.Controls(player: VideoPlayer) {
val playing = remember(player) { player.videoPlaying }
val progress = remember(player) { player.progress }
val duration = remember(player) { player.duration }
@ -62,10 +70,7 @@ private fun BoxScope.Controls(player: VideoPlayer, close: () -> Unit) {
Slider(
value = progress.value.toFloat() / max(0.0001f, duration.value.toFloat()),
onValueChange = { player.player.seekTo((it * duration.value).toInt()) },
modifier = Modifier.fillMaxWidth().weight(1f)
modifier = Modifier.fillMaxWidth()
)
IconButton(onClick = close,) {
Icon(painterResource(MR.images.ic_close), null, Modifier.size(30.dp), tint = MaterialTheme.colors.primary)
}
}
}