camera2 captured picture - conversion from YUV_420_888 to NV21

Alex Cohn :

Your code will only return correct NV21 if there is no padding at all, and U and V plains overlap and actually represent interlaced VU values. This happens quite often for preview, but in such case you allocate extra w*h/4 bytes for your array (which presumably is not a problem). Maybe for captured image you need a more robust implemenation, e.g.\nprivate static byte[] YUV_420_888toNV21(Image image) {\n\n int width = image.getWidth();\n int height = image.getHeight(); \n int ySize = width*height;\n int uvSize = width*height/4;\n\n byte[] nv21 = new byte[ySize + uvSize*2];\n\n ByteBuffer yBuffer = image.getPlanes()[0].getBuffer(); // Y\n ByteBuffer uBuffer = image.getPlanes()[1].getBuffer(); // U\n ByteBuffer vBuffer = image.getPlanes()[2].getBuffer(); // V\n\n int rowStride = image.getPlanes()[0].getRowStride();\n assert(image.getPlanes()[0].getPixelStride() == 1);\n\n int pos = 0;\n\n if (rowStride == width) { // likely\n yBuffer.get(nv21, 0, ySize);\n pos += ySize;\n }\n else {\n long yBufferPos = -rowStride; // not an actual position\n for (; pos<ySize; pos+=width) {\n yBufferPos += rowStride;\n yBuffer.position(yBufferPos);\n yBuffer.get(nv21, pos, width);\n }\n }\n\n rowStride = image.getPlanes()[2].getRowStride();\n int pixelStride = image.getPlanes()[2].getPixelStride();\n\n assert(rowStride == image.getPlanes()[1].getRowStride());\n assert(pixelStride == image.getPlanes()[1].getPixelStride());\n \n if (pixelStride == 2 && rowStride == width && uBuffer.get(0) == vBuffer.get(1)) {\n // maybe V an U planes overlap as per NV21, which means vBuffer[1] is alias of uBuffer[0]\n byte savePixel = vBuffer.get(1);\n try {\n vBuffer.put(1, (byte)~savePixel);\n if (uBuffer.get(0) == (byte)~savePixel) {\n vBuffer.put(1, savePixel);\n vBuffer.position(0);\n uBuffer.position(0);\n vBuffer.get(nv21, ySize, 1);\n uBuffer.get(nv21, ySize + 1, uBuffer.remaining());\n\n return nv21; // shortcut\n }\n }\n catch (ReadOnlyBufferException ex) {\n // unfortunately, we cannot check if vBuffer and uBuffer overlap\n }\n\n // unfortunately, the check failed. We must save U and V pixel by pixel\n vBuffer.put(1, savePixel);\n }\n\n // other optimizations could check if (pixelStride == 1) or (pixelStride == 2), \n // but performance gain would be less significant\n\n for (int row=0; row<height/2; row++) {\n for (int col=0; col<width/2; col++) {\n int vuPos = col*pixelStride + row*rowStride;\n nv21[pos++] = vBuffer.get(vuPos);\n nv21[pos++] = uBuffer.get(vuPos);\n }\n }\n\n return nv21;\n}\n\nIf you anyway intend to pass the resulting array to C++, you can take advantage of the fact that\n\nthe buffer returned will always have isDirect return true, so the underlying data could be mapped as a pointer in JNI without doing any copies with GetDirectBufferAddress.\n\nThis means that same conversion may be done in C++ with minimal overhead. In C++, you may even find that the actual pixel arrangement is already NV21!\nPS Actually, this can be done in Java, with negligible overhead, see the line if (pixelStride == 2 && … above. So, we can bulk copy all chroma bytes to the resulting byte array, which is much faster than running the loops, but still slower than what can be achieved for such case in C++. For full implementation, see Image.toByteArray().",

2018-10-10T12:54:15

Juan Miguel S. :

Based on @Alex Cohn answer, I have implemented it in the JNI part, trying to take profit from the byte-access and performance advantages. I left it here, maybe it could be as useful as the @Alex answer was for me. It's almost the same algorithm, in C; based in an image with YUV_420_888 format:\n\nuchar* yuvToNV21(jbyteArray yBuf, jbyteArray uBuf, jbyteArray vBuf, jbyte *fullArrayNV21,\n int width, int height, int yRowStride, int yPixelStride, int uRowStride,\n int uPixelStride, int vRowStride, int vPixelStride, JNIEnv *env) {\n\n /* Check that our frame has right format, as specified at android docs for\n * YUV_420_888 (https://developer.android.com/reference/android/graphics/ImageFormat?authuser=2#YUV_420_888):\n * - Plane Y not overlaped with UV, and always with pixelStride = 1\n * - Planes U and V have the same rowStride and pixelStride (overlaped or not)\n */\n if(yPixelStride != 1 || uPixelStride != vPixelStride || uRowStride != vRowStride) {\n jclass Exception = env->FindClass(\"java/lang/Exception\");\n env->ThrowNew(Exception, \"Invalid YUV_420_888 byte structure. Not agree with https://developer.android.com/reference/android/graphics/ImageFormat?authuser=2#YUV_420_888\");\n }\n\n int ySize = width*height;\n int uSize = env->GetArrayLength(uBuf);\n int vSize = env->GetArrayLength(vBuf);\n int newArrayPosition = 0; //Posicion por la que vamos rellenando el array NV21\n if (fullArrayNV21 == nullptr) {\n fullArrayNV21 = new jbyte[ySize + uSize + vSize];\n }\n if(yRowStride == width) {\n //Best case. No padding, copy direct\n env->GetByteArrayRegion(yBuf, newArrayPosition, ySize, fullArrayNV21);\n newArrayPosition = ySize;\n }else {\n // Padding at plane Y. Copy Row by Row\n long yPlanePosition = 0;\n for(; newArrayPosition<ySize; newArrayPosition += width) {\n env->GetByteArrayRegion(yBuf, yPlanePosition, width, fullArrayNV21 + newArrayPosition);\n yPlanePosition += yRowStride;\n }\n }\n\n // Check UV channels in order to know if they are overlapped (best case)\n // If they are overlapped, U and B first bytes are consecutives and pixelStride = 2\n long uMemoryAdd = (long)&uBuf;\n long vMemoryAdd = (long)&vBuf;\n long diff = std::abs(uMemoryAdd - vMemoryAdd);\n if(vPixelStride == 2 && diff == 8) {\n if(width == vRowStride) {\n // Best Case: Valid NV21 representation (UV overlapped, no padding). Copy direct\n env->GetByteArrayRegion(uBuf, 0, uSize, fullArrayNV21 + ySize);\n env->GetByteArrayRegion(vBuf, 0, vSize, fullArrayNV21 + ySize + uSize);\n }else {\n // UV overlapped, but with padding. Copy row by row (too much performance improvement compared with copy byte-by-byte)\n int limit = height/2 - 1;\n for(int row = 0; row<limit; row++) {\n env->GetByteArrayRegion(uBuf, row * vRowStride, width, fullArrayNV21 + ySize + (row * width));\n }\n }\n }else {\n //WORST: not overlapped UV. Copy byte by byte\n for(int row = 0; row<height/2; row++) {\n for(int col = 0; col<width/2; col++) {\n int vuPos = col*uPixelStride + row*uRowStride;\n env->GetByteArrayRegion(vBuf, vuPos, 1, fullArrayNV21 + newArrayPosition);\n newArrayPosition++;\n env->GetByteArrayRegion(uBuf, vuPos, 1, fullArrayNV21 + newArrayPosition);\n newArrayPosition++;\n }\n }\n }\n return (uchar*)fullArrayNV21;\n}\n\n\nI'm sure that some improvements can be added, but I have tested in a lot of devices, and it is working with very good performance and stability.",

2020-05-19T10:26:52

Max Filinski :

public static byte[] YUV420toNV21(Image image) {\n Rect crop = image.getCropRect();\n int format = image.getFormat();\n int width = crop.width();\n int height = crop.height();\n Image.Plane[] planes = image.getPlanes();\n byte[] data = new byte[width * height * ImageFormat.getBitsPerPixel(format) / 8];\n byte[] rowData = new byte[planes[0].getRowStride()];\n\n int channelOffset = 0;\n int outputStride = 1;\n for (int i = 0; i < planes.length; i++) {\n switch (i) {\n case 0:\n channelOffset = 0;\n outputStride = 1;\n break;\n case 1:\n channelOffset = width * height + 1;\n outputStride = 2;\n break;\n case 2:\n channelOffset = width * height;\n outputStride = 2;\n break;\n }\n\n ByteBuffer buffer = planes[i].getBuffer();\n int rowStride = planes[i].getRowStride();\n int pixelStride = planes[i].getPixelStride();\n\n int shift = (i == 0) ? 0 : 1;\n int w = width >> shift;\n int h = height >> shift;\n buffer.position(rowStride * (crop.top >> shift) + pixelStride * (crop.left >> shift));\n for (int row = 0; row < h; row++) {\n int length;\n if (pixelStride == 1 && outputStride == 1) {\n length = w;\n buffer.get(data, channelOffset, length);\n channelOffset += length;\n } else {\n length = (w - 1) * pixelStride + 1;\n buffer.get(rowData, 0, length);\n for (int col = 0; col < w; col++) {\n data[channelOffset] = rowData[col * pixelStride];\n channelOffset += outputStride;\n }\n }\n if (row < h - 1) {\n buffer.position(buffer.position() + rowStride - length);\n }\n }\n }\n return data;\n }\n",

2019-12-13T10:48:45