目录

How do I convert a PixelMap received on the C++ side into the cv::Mat format?

Solution

You can use either of the following methods to convert a PixelMap into cv::Mat:

In both methods, ensure that the format of the PixelMap is the same as that of Mat in OpenCV. Otherwise, color deviation occurs.

Refer to the code snippet below.

Parameters passed by ArkTS:

import cPixelMapToMat from 'libcpixelmaptomat.so';
import { BusinessError } from '@kit.BasicServicesKit';
import { image } from '@kit.ImageKit';

@Entry
@Component
struct Index {
  @State pixelMap: image.PixelMap | undefined = undefined

  async arrayBufferToMat() {
    if (this.pixelMap == undefined || this.pixelMap){
      let context = this.getUIContext().getHostContext() as common.UIAbilityContext;
      let resourceManager = context.resourceManager
      let imageArray = await resourceManager.getMediaContent($r('app.media.sample10'));
      let pixelBuffer = new Uint8Array(imageArray).buffer as Object as ArrayBuffer
      console.info("pixelBuffer length: " + pixelBuffer.byteLength);
      let imageResource = image.createImageSource(pixelBuffer);
      let opts: image.DecodingOptions = {
        editable: true,
        desiredPixelFormat: image.PixelMapFormat.RGBA_8888
      }
      this.pixelMap = await imageResource.createPixelMap(opts);
    }

    const readBuffer: ArrayBuffer = new ArrayBuffer(this.pixelMap.getPixelBytesNumber()); // Obtain the array buffer of the pixelmap
    console.info("readBuffer length: " + readBuffer.byteLength);
    this.pixelMap.readPixelsToBuffer(readBuffer).then(() => {
      console.info("No Error!")
    }).catch((err: BusinessError) => {
      console.error("Error! " + err.message)
    })
    const dir = getContext(this).filesDir;
    console.info('save path: ' + dir);
    cPixelMapToMat.arrayBufferToMat(this.pixelMap, readBuffer, dir);
  }

  async accessToMat(){
    if (this.pixelMap == undefined || this.pixelMap) {
      let resourceManager = getContext(this).resourceManager
      let imageArray = await resourceManager.getMediaContent($r('app.media.sample14'));
      let pixelBuffer = new Uint8Array(imageArray).buffer as Object as ArrayBuffer
      console.info("pixelBuffer length: " + pixelBuffer.byteLength);
      let imageResource = image.createImageSource(pixelBuffer);
      let opts: image.DecodingOptions = {
        editable: true,
        desiredPixelFormat: image.PixelMapFormat.RGBA_8888
      }
      this.pixelMap = await imageResource.createPixelMap(opts);
    }

    const dir = getContext(this).filesDir;
    console.info('save path: ' + dir);
    cPixelMapToMat.accessToMat(this.pixelMap, dir);
  }

  build() {
    Row() {
      Column() {
        Image(this.pixelMap)
          .width(200)
          .height(200)
        Button('ArrayBufferToMat')
          .onClick(() => {
            this.arrayBufferToMat();
          })

        Button('AccessToMat')
          .onClick(() => {
            this.accessToMat();
          })
      }
      .width('100%')
    }
    .height('100%')
  }
}

Solution 1: Converting an ArrayBuffer into cv::Mat:

#include "napi/native_api.h"
#include <multimedia/image_framework/image_mdk.h>
#include <multimedia/image_framework/image_mdk_common.h>
#include <multimedia/image_framework/image_pixel_map_mdk.h>
#include <multimedia/image_framework/image_pixel_map_napi.h>
#include "hilog/log.h"
#include <opencv2/opencv.hpp>
#include <bits/alltypes.h>

static napi_value ArrayBufferToMat(napi_env env, napi_callback_info info) {
    size_t argc = 3;
    napi_value args[3] = {nullptr};
    napi_get_cb_info(env, info, &argc, args, nullptr, nullptr);

    napi_value error;
    napi_create_int32(env, -1, &error);

    // Initialize PixelMap object data 
    NativePixelMap *native = OH_PixelMap_InitNativePixelMap(env, args[0]);
    if (native == nullptr) {
        return error;
    }
    // Obtaining Image Information
    struct OhosPixelMapInfos pixelMapInfos;
    if (OH_PixelMap_GetImageInfo(native, &pixelMapInfos) != IMAGE_RESULT_SUCCESS) {
        OH_LOG_Print(LOG_APP, LOG_ERROR, 0xFF00, "Test", "Pure : -1");
        return error;
    }
    // Obtains the buffer
    napi_value buffer = args[1];
    napi_valuetype valueType;
    napi_typeof(env, buffer, &valueType);
    if (valueType == napi_object) {
        bool isArrayBuffer = false;
        napi_is_arraybuffer(env, buffer, &isArrayBuffer);
        if (!isArrayBuffer) {
            napi_throw_error(env, "EINVAL", "Error");
        }
    }

    void *data = nullptr;
    size_t byteLength = 0;
    napi_get_arraybuffer_info(env, buffer, &data, &byteLength);
    int32_t *saveBuffer = (int32_t *)(data);

    // Convert to Mat
    cv::Mat originMat(pixelMapInfos.height, pixelMapInfos.width, CV_8UC4, saveBuffer);
    if (!originMat.data) {
        OH_LOG_Print(LOG_APP, LOG_ERROR, 0xFF00, "Read Image", "Pure : -1");
        return error;
    }

    // openCV defaults to BGRA or BGR. If the pixelmap is not created in one of these formats, a format conversion is required
    cv::Mat saveMat;
    cv::cvtColor(originMat, saveMat, cv::COLOR_BGRA2RGBA);
    char pathArray[1024];
    size_t length;
    napi_get_value_string_utf8(env, args[2], pathArray, 1024, &length);
    std::string path(pathArray);
    path += "/buffer.jpg";
    if (!cv::imwrite(path, saveMat)) {
        OH_LOG_Print(LOG_APP, LOG_ERROR, 0xFF00, "Write Image", "Pure : -1");
        return error;
    }

    napi_value res;
    napi_create_int32(env, 1, &res);
    return res;
}

Solution 2: Converting the data in the PixelMap memory address, which is obtained by calling OH_PixelMap_AccessPixels, into cv::Mat:

static napi_value AccessToMat(napi_env env, napi_callback_info info) {
    size_t argc = 2;
    napi_value args[2] = {nullptr};
    napi_get_cb_info(env, info, &argc, args, nullptr, nullptr);

    napi_value error;
    napi_create_int32(env, -1, &error);

    NativePixelMap *native = OH_PixelMap_InitNativePixelMap(env, args[0]);
    if (native == nullptr) {
        return error;
    }
    struct OhosPixelMapInfos pixelMapInfos;
    if (OH_PixelMap_GetImageInfo(native, &pixelMapInfos) != IMAGE_RESULT_SUCCESS) {
        OH_LOG_Print(LOG_APP, LOG_ERROR, 0xFF00, "Test", "Pure : -1");
        return error;
    }

    void *pixel;
    // Obtain the memory address of the NativePixelMap object and lock the memory
    OH_PixelMap_AccessPixels(native, &pixel);

    // Convert to Mat, pay attention to alignment, so rowSize needs to be passed in
    cv::Mat originMat(pixelMapInfos.height, pixelMapInfos.width, CV_8UC4, pixel, pixelMapInfos.rowSize);
    if (!originMat.data) {
        OH_LOG_Print(LOG_APP, LOG_ERROR, 0xFF00, "Read Image", "Pure : -1");
        return error;
    }

    // openCV defaults to BGRA or BGR. If the pixelmap is not created in one of these formats, a format conversion is required
    cv::Mat saveMat;
    cv::cvtColor(originMat, saveMat, cv::COLOR_BGRA2RGBA);
    char pathArray[1024];
    size_t length;
    napi_get_value_string_utf8(env, args[1], pathArray, 1024, &length);
    std::string path(pathArray);
    path += "/access.jpg";
    if (!cv::imwrite(path, saveMat)) {
        OH_LOG_Print(LOG_APP, LOG_ERROR, 0xFF00, "Write Image", "Pure : -1");
        return error;
    }

    napi_value res;
    napi_create_int32(env, 1, &res);
    return res;
}

In HarmonyOS development, the memory size needs to be specified for images that support hardware decoding. OH_PixelMap_AccessPixels() obtains the memory address of the image and locks the memory. However, the actual image size needs to be aligned with lineStride. Therefore, when constructing a Mat object, you must specify lineStride, that is, rowSize, for alignment. You can call OH_GetImageInfo to obtain the image information, including the width, height, and row size.