Android camera preview rendered to a texture is stretched

Question

I'm sending android camera preview to Unity3d using texture and it is stretched. The preview is not rendered to any view in android but directly to a texture with:

setPreviewTexture(texture);

Then, the bytes are sent to Unity3d and drawn on screen with every onPreviewFrame.

The camera preview size and the texture size are set to 1024x768 and the texture container in Unity3d has the same size. The device's resolution is 960x540 so it's a different ratio. I'm selecting a supported resolution for the camera, the native size has a ratio of 4:3 so there should be no stretching. It seems as if android is rendering to texture only the part that can actually be rendered to screen - it takes a 16:9 image and renders it to a 4:3 texture. If I'm wrong please correct me but that how it seems to work in this example. Here is a bit of code:

public int startCamera(int idx, int width, int height) {
    nativeTexturePointer = createExternalTexture();
    texture = new SurfaceTexture(nativeTexturePointer);

    mCamera = Camera.open(idx);
    setupCamera(width, height);

    try {
        mCamera.setPreviewTexture(texture);
        mCamera.setPreviewCallback(this);
        mCamera.startPreview();
        Log.i("Unity", "JAVA: camera started");
    } catch (IOException ioe) {
        Log.w("Unity", "JAVA: CAM LAUNCH FAILED");
    }
    return nativeTexturePointer;
}

public void stopCamera() {
    mCamera.setPreviewCallback(null);
    mCamera.stopPreview();
    mCamera.release();
    GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, 0);
    Log.i("Unity", "JAVA: Camera stopped");
}

private int createExternalTexture() {
    int[] textureIdContainer = new int[1];
    GLES20.glGenTextures(1, textureIdContainer, 0);
    GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, textureIdContainer[0]);
    return textureIdContainer[0];
}

@SuppressLint("NewApi")
private void setupCamera(int width, int height) {
    Camera.Parameters params = mCamera.getParameters();
    params.setRecordingHint(true);
    params.setFocusMode(Camera.Parameters.FOCUS_MODE_CONTINUOUS_VIDEO);
    params.setPreviewFormat(17);
    params.setZoom(0);
        // 16 ~ NV16 ~ YCbCr
        // 17 ~ NV21 ~ YCbCr ~ DEFAULT *
        // 4  ~ RGB_565
        // 256~ JPEG
        // 20 ~ YUY2 ~ YcbCr ...
        // 842094169 ~ YV12 ~ 4:2:0 YCrCb comprised of WXH Y plane, W/2xH/2 Cr & Cb. see documentation *
    params.setFlashMode(Camera.Parameters.FLASH_MODE_TORCH);
    Camera.Size previewSize = getOptimalSize(width, height, mCamera.getParameters().getSupportedPreviewSizes());
    Camera.Size picSize = getOptimalSize(width, height, mCamera.getParameters().getSupportedPictureSizes());

    params.setPictureSize(picSize.width, picSize.height);
    params.setPreviewSize(previewSize.width, previewSize.height);
    params.setWhiteBalance(Camera.Parameters.WHITE_BALANCE_AUTO);
    params.setExposureCompensation(0);

    try{
        mCamera.setParameters(params);
    } catch (Exception e){
        Log.i("Unity", "ERROR: " + e.getMessage());
    }

    Camera.Size mCameraPreviewSize = params.getPreviewSize();
    prevWidth = mCameraPreviewSize.width;
    prevHeight = mCameraPreviewSize.height;

    int[] fpsRange = new int[2];
    params.getPreviewFpsRange(fpsRange);
}

private Camera.Size getOptimalSize(int width, int height, List<Camera.Size> sizes) {
    if(mCamera == null)
        return null;

    final double ASPECT_TOLERANCE = 0.1;
    double targetRatio=(double)width / height;

    if (sizes == null)
        return null;

    Camera.Size optimalSize = null;
    double minDiff = Double.MAX_VALUE;
    int targetWidth = width;

    for (Camera.Size size : sizes) {
        double ratio = (double) size.width / size.height;
        Log.i("Unity", "RES: size=" + size.width + "/" + size.height + "/ Aspect Ratio: " + ratio + "target width: " + width + "target height: " + height);

        if (Math.abs(ratio - targetRatio) > ASPECT_TOLERANCE) continue;
        if (Math.abs(size.width - targetWidth) < minDiff) {
            optimalSize = size;
            minDiff = Math.abs(size.width - targetWidth);
        }
    }

    if (optimalSize == null) {
        minDiff = Double.MAX_VALUE;
        for (Camera.Size size : sizes) {
            if (Math.abs(size.width - targetWidth) < minDiff) {
                optimalSize = size;
                minDiff = Math.abs(size.width - targetWidth);
            }
        }
    }
    Log.i("Unity", "optimal size=" + optimalSize.width + "/" + optimalSize.height + "/ Aspect Ratio: " + (double) optimalSize.width / optimalSize.height);
    return optimalSize;
}

public int getPreviewSizeWidth() {
    return prevWidth;
}

public int getPreviewSizeHeight() { return prevHeight; }

public byte[] bytes;
@Override
public void onPreviewFrame(byte[] data, Camera camera) {
    bytes = data;

    //Log.i("Unity", "JAVA: " + Integer.toString(data.length));
    UnityPlayer.UnitySendMessage(gameObjectTargetName, "GetBuffer", "");
}

It is required that the texture sent to Unity3d is 1024x768 without cropping. Does anyone have any fix for that?

UPDATE(added C# code)

This code gets preview from java:

void CreateCameraTexture() {
    _texWidth = nativeCameraObject.Call<int>("getPreviewSizeWidth");
    _texHeight = nativeCameraObject.Call<int>("getPreviewSizeHeight");
    _cameraPreview = new Texture2D(_texWidth, _texHeight, TextureFormat.Alpha8, false);
    _converter = new YUVDecode(_texWidth, _texHeight);
    shaderMat.SetFloat("_Width", _texWidth);

    if (OnCameraTextureCreated != null)
        OnCameraTextureCreated(_cameraPreview);
}

private byte[] _bytes;
public void GetBuffer(string str) {
    _bytes = nativeCameraObject.Get<byte[]>("bytes");
    SetColor(_bytes);
    _cameraPreview.LoadRawTextureData(_bytes);
    _cameraPreview.Apply();

    UpdateLibFrame();
}

void SetColor(byte[] bytes) {
    _converter.SetBytes(bytes);
    shaderMat.SetTexture("_YUVTex", _converter.yuvtexture);
}

void UpdateLibFrame() {
    if (OnFrameUpdate != null) OnFrameUpdate(_cameraPreview.GetRawTextureData());
}

This code in another class shows the data:

private RectTransform previewRect {
    get {
        if (!_previewRect)
            _previewRect = cameraPreview.GetComponent<RectTransform>();
        return _previewRect;
    }
}

private void GetPreviewFromCamera() {
    NativeCamera.OnCameraTextureCreated += (Texture2D cameraTex) => {
        if (cameraPreview != null && cameraPreview.texture != null)
            DestroyImmediate(cameraPreview.texture);

        previewRect.sizeDelta = new Vector2(1024, 768);

        cameraPreview.texture = cameraTex;
        cameraPreview.enabled = true;
    };
}

UPDATE: Added link to a sample project

The project is here: https://www.dropbox.com/s/wid1qa9cmq3ck6w/CameraPreview.zip?dl=0

CameraJava is a gradle project. The output is a .AAR file that you have to copy to Assets/Plugins.

Answer 1

sorry for the delay. i finally had time to test your project. it looks fine to me. here is what you can do in Unity to get a good result (though only tested for landscape, since you haven't implemented rotation handling in android it seems): go to player settings and set default orientation to landscape left. set the Canvas Scaler to the same reference resolution as your camera image (1024x768). finally set Z rotation of RawImage to 0. due to the 4:3 ratio, top and bottom will be cut but that's normal. if you want the full image, set the cavas scaler to match height instead of width.