CRYPT_SIGN_MESSAGE_PARA param;
    DWORD MessageSizeArray[1];
    const BYTE* MessageArray[1];
    DWORD signed_len = 0;
    BYTE *signed_mem = NULL;
....

    ZeroMemory (&param, sizeof (param));
    param.cbSize = sizeof(CRYPT_SIGN_MESSAGE_PARA);
    param.dwMsgEncodingType = TYPE_DER;
    param.pSigningCert = pUserCert;
    
    param.HashAlgorithm.pszObjId = OID; /*szOID_RSA_MD5;*/
    param.HashAlgorithm.Parameters.cbData = 0;
    param.HashAlgorithm.Parameters.pbData = NULL;
    
    param.pvHashAuxInfo = NULL;

...
 param.dwFlags = 0;
    param.rgAuthAttr = NULL;
    
    MessageArray[0] = mem_tbs;
    MessageSizeArray[0] = mem_len;

    ret = CryptSignMessage(
	&param,
	detached,
	1,  /* количество detached или обоих (всегда тогда 1) вложений*/
	MessageArray,
	MessageSizeArray,
	NULL,
	&signed_len);

    if (ret) {
	printf("Calculate message len OK \n");
    }
    else
    {
	HandleErrorFL("Calculate message len");
    }