我试图将virtualDisplay的结果呈现为opengl es中的纹理，以在其上应用着色器。所有这些都是在后台服务中完成的。
我已遵循此演示：Using SurfaceTexture in Android
并以Grafika examples结尾的代码结尾：
GL Surface视图类和渲染器
package com.techMeUp.phenixtv
import android.annotation.SuppressLint
import android.content.Context
import android.graphics.SurfaceTexture
import android.hardware.display.VirtualDisplay
import android.opengl.GLES20
import android.opengl.GLSurfaceView
import android.opengl.Matrix
import android.util.Log
import android.view.Surface
import java.nio.ByteBuffer
import java.nio.ByteOrder
import java.nio.FloatBuffer
import javax.microedition.khronos.egl.EGLConfig
import javax.microedition.khronos.opengles.GL10
@SuppressLint("ViewConstructor")
class OverlaySurfaceView(context: Context, mv: VirtualDisplay) : GLSurfaceView(context) {
private var mRenderer: OverlayRender
private var mVirtualDisplay: VirtualDisplay
init {
setEGLContextClientVersion(2)
mVirtualDisplay = mv
mRenderer = OverlayRender()
setRenderer(mRenderer)
}
override fun onResume() {
queueEvent { mRenderer.setVirtualDisplay(mVirtualDisplay) }
super.onResume()
}
private class OverlayRender : Renderer, SurfaceTexture.OnFrameAvailableListener {
private val mTriangleVerticesData = floatArrayOf(
// X, Y, Z, U, V
-1.0f, -1.0f, 0f, 0f, 0f, 1.0f, -1.0f, 0f, 1f, 0f, -1.0f, 1.0f, 0f, 0f, 1f, 1.0f, 1.0f, 0f, 1f, 1f
)
private val mTriangleVertices: FloatBuffer
private val mVertexShader = "uniform mat4 uMVPMatrix;\n" +
"uniform mat4 uSTMatrix;\n" +
"attribute vec4 aPosition;\n" +
"attribute vec4 aTextureCoord;\n" +
"varying vec2 vTextureCoord;\n" +
"void main() {\n" +
" gl_Position = uMVPMatrix * aPosition;\n" +
" vTextureCoord = (uSTMatrix * aTextureCoord).xy;\n" +
"}\n"
private val mFragmentShader = "#extension GL_OES_EGL_image_external : require\n" +
"precision mediump float;\n" +
"varying vec2 vTextureCoord;\n" +
"uniform samplerExternalOES sTexture;\n" +
"void main() {\n" +
" gl_FragColor = texture2D(sTexture, vTextureCoord);\n" +
"}\n"
private val mMVPMatrix = FloatArray(16)
private val mSTMatrix = FloatArray(16)
private var mProgram: Int = 0
private var mTextureID: Int = 0
private var muMVPMatrixHandle: Int = 0
private var muSTMatrixHandle: Int = 0
private var maPositionHandle: Int = 0
private var maTextureHandle: Int = 0
private var mSurface: SurfaceTexture? = null
private var updateSurface = false
private var mVirtualDisplay: VirtualDisplay? = null
init {
mTriangleVertices = ByteBuffer.allocateDirect(
mTriangleVerticesData.size * FLOAT_SIZE_BYTES
)
.order(ByteOrder.nativeOrder()).asFloatBuffer()
mTriangleVertices.put(mTriangleVerticesData).position(0)
Matrix.setIdentityM(mSTMatrix, 0)
}
fun setVirtualDisplay(display: VirtualDisplay) {
mVirtualDisplay = display
}
override fun onDrawFrame(glUnused: GL10) {
synchronized(this) {
if (updateSurface) {
mSurface!!.updateTexImage()
mSurface!!.getTransformMatrix(mSTMatrix)
updateSurface = false
}
}
GLES20.glClearColor(0.0f, 0.0f, 0.0f, 1.0f)
GLES20.glClear(GLES20.GL_DEPTH_BUFFER_BIT or GLES20.GL_COLOR_BUFFER_BIT)
GLES20.glUseProgram(mProgram)
checkGlError("glUseProgram")
GLES20.glActiveTexture(GLES20.GL_TEXTURE0)
GLES20.glBindTexture(GL_TEXTURE_EXTERNAL_OES, mTextureID)
mTriangleVertices.position(TRIANGLE_VERTICES_DATA_POS_OFFSET)
GLES20.glVertexAttribPointer(
maPositionHandle, 3, GLES20.GL_FLOAT, false,
TRIANGLE_VERTICES_DATA_STRIDE_BYTES, mTriangleVertices
)
checkGlError("glVertexAttribPointer maPosition")
GLES20.glEnableVertexAttribArray(maPositionHandle)
checkGlError("glEnableVertexAttribArray maPositionHandle")
mTriangleVertices.position(TRIANGLE_VERTICES_DATA_UV_OFFSET)
GLES20.glVertexAttribPointer(
maTextureHandle, 3, GLES20.GL_FLOAT, false,
TRIANGLE_VERTICES_DATA_STRIDE_BYTES, mTriangleVertices
)
checkGlError("glVertexAttribPointer maTextureHandle")
GLES20.glEnableVertexAttribArray(maTextureHandle)
checkGlError("glEnableVertexAttribArray maTextureHandle")
Matrix.setIdentityM(mMVPMatrix, 0)
GLES20.glUniformMatrix4fv(muMVPMatrixHandle, 1, false, mMVPMatrix, 0)
GLES20.glUniformMatrix4fv(muSTMatrixHandle, 1, false, mSTMatrix, 0)
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4)
checkGlError("glDrawArrays")
GLES20.glFinish()
}
override fun onSurfaceChanged(glUnused: GL10, width: Int, height: Int) {
}
override fun onSurfaceCreated(glUnused: GL10, config: EGLConfig) {
mProgram = createProgram(mVertexShader, mFragmentShader)
if (mProgram == 0) {
return
}
maPositionHandle = GLES20.glGetAttribLocation(mProgram, "aPosition")
checkGlError("glGetAttribLocation aPosition")
if (maPositionHandle == -1) {
throw RuntimeException("Could not get attrib location for aPosition")
}
maTextureHandle = GLES20.glGetAttribLocation(mProgram, "aTextureCoord")
checkGlError("glGetAttribLocation aTextureCoord")
if (maTextureHandle == -1) {
throw RuntimeException("Could not get attrib location for aTextureCoord")
}
muMVPMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uMVPMatrix")
checkGlError("glGetUniformLocation uMVPMatrix")
if (muMVPMatrixHandle == -1) {
throw RuntimeException("Could not get attrib location for uMVPMatrix")
}
muSTMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uSTMatrix")
checkGlError("glGetUniformLocation uSTMatrix")
if (muSTMatrixHandle == -1) {
throw RuntimeException("Could not get attrib location for uSTMatrix")
}
val textures = IntArray(1)
GLES20.glGenTextures(1, textures, 0)
mTextureID = textures[0]
GLES20.glBindTexture(GL_TEXTURE_EXTERNAL_OES, mTextureID)
checkGlError("glBindTexture mTextureID")
GLES20.glTexParameterf(
GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER,
GLES20.GL_NEAREST.toFloat()
)
GLES20.glTexParameterf(
GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER,
GLES20.GL_LINEAR.toFloat()
)
/*
* Create the SurfaceTexture that will feed this textureID,
* and pass it to the MediaPlayer
*/
mSurface = SurfaceTexture(mTextureID)
mSurface!!.setOnFrameAvailableListener(this)
val surface = Surface(mSurface)
mVirtualDisplay!!.surface = surface
surface.release()
synchronized(this) {
updateSurface = false
}
}
@Synchronized
override fun onFrameAvailable(surface: SurfaceTexture) {
updateSurface = true
}
private fun loadShader(shaderType: Int, source: String): Int {
var shader = GLES20.glCreateShader(shaderType)
if (shader != 0) {
GLES20.glShaderSource(shader, source)
GLES20.glCompileShader(shader)
val compiled = IntArray(1)
GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, compiled, 0)
if (compiled[0] == 0) {
Log.e(TAG, "Could not compile shader $shaderType:")
Log.e(TAG, GLES20.glGetShaderInfoLog(shader))
GLES20.glDeleteShader(shader)
shader = 0
}
}
return shader
}
private fun createProgram(vertexSource: String, fragmentSource: String): Int {
val vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, vertexSource)
if (vertexShader == 0) {
return 0
}
val pixelShader = loadShader(GLES20.GL_FRAGMENT_SHADER, fragmentSource)
if (pixelShader == 0) {
return 0
}
var program = GLES20.glCreateProgram()
if (program != 0) {
GLES20.glAttachShader(program, vertexShader)
checkGlError("glAttachShader")
GLES20.glAttachShader(program, pixelShader)
checkGlError("glAttachShader")
GLES20.glLinkProgram(program)
val linkStatus = IntArray(1)
GLES20.glGetProgramiv(program, GLES20.GL_LINK_STATUS, linkStatus, 0)
if (linkStatus[0] != GLES20.GL_TRUE) {
Log.e(TAG, "Could not link program: ")
Log.e(TAG, GLES20.glGetProgramInfoLog(program))
GLES20.glDeleteProgram(program)
program = 0
}
}
return program
}
private fun checkGlError(op: String) {
val error = GLES20.glGetError()
while ((error) != GLES20.GL_NO_ERROR) {
Log.e(TAG, "$op: glError $error")
throw RuntimeException("$op: glError $error")
}
}
companion object {
private const val TAG = "VideoRender"
private const val FLOAT_SIZE_BYTES = 4
private const val TRIANGLE_VERTICES_DATA_STRIDE_BYTES = 5 * FLOAT_SIZE_BYTES
private const val TRIANGLE_VERTICES_DATA_POS_OFFSET = 0
private const val TRIANGLE_VERTICES_DATA_UV_OFFSET = 3
private const val GL_TEXTURE_EXTERNAL_OES = 0x8D65
}
}
}
在我的后台服务中创建VirtualDisplay和GLSurfaceView
val projection = manager.getMediaProjection(resultCode, resultData) as MediaProjection
val mVirtualDisplay = projection.createVirtualDisplay("virtualdisplay",
resources.displayMetrics.widthPixels, resources.displayMetrics.heightPixels, resources.displayMetrics.densityDpi,
DisplayManager.VIRTUAL_DISPLAY_FLAG_AUTO_MIRROR,
null, null, null)
mOverlaySurfaceView = OverlaySurfaceView(this, mVirtualDisplay)
val type = WindowManager.LayoutParams.TYPE_APPLICATION_OVERLAY
val params = WindowManager.LayoutParams(
type,
WindowManager.LayoutParams.FLAG_NOT_TOUCHABLE or WindowManager.LayoutParams.FLAG_LAYOUT_IN_SCREEN or WindowManager.LayoutParams.FLAG_NOT_FOCUSABLE,
PixelFormat.TRANSLUCENT
)
wm = getSystemService(Context.WINDOW_SERVICE) as WindowManager
wm.addView(mOverlaySurfaceView, params)
没有错误消息，但纹理显示为灰色屏幕，而不是实际的虚拟显示。 虚拟显示似乎可以正常工作，因为我已经在非GL surfaceView上进行了显示。
我认为将纹理绑定到openGL存在问题，但是我无法找到我应该更改的参数才能完成所有这些工作。