实现数字人视频通话

Express

- (BOOL)enableCustomVideoRender {
    // 自定义渲染
    ZegoCustomVideoRenderConfig *renderConfig =
    [[ZegoCustomVideoRenderConfig alloc] init];
    // 选择 RawData 类型视频帧数据
    renderConfig.bufferType = ZegoVideoBufferTypeRawData;
    // 选择 RGB 色系数据格式
    renderConfig.frameFormatSeries = ZegoVideoFrameFormatSeriesRGB;
    // 指定在自定义视频渲染的同时引擎也渲染
    renderConfig.enableEngineRender = NO;

    ZegoExpressEngine *engine = [ZegoExpressEngine sharedEngine];
    if (!engine) {
        return NO;
    }

    [engine enableCustomVideoRender:YES config:renderConfig];
    [engine setCustomVideoRenderHandler:self];

    return YES;
}

#pragma mark - ZegoEventHandler

- (void)onRemoteVideoFrameRawData:(unsigned char **)data
                       dataLength:(unsigned int *)dataLength
                            param:(ZegoVideoFrameParam *)param
                         streamID:(NSString *)streamID {
    // 转换参数格式
    ZDMVideoFrameParam *digitalParam = [[ZDMVideoFrameParam alloc] init];
    digitalParam.format = (ZDMVideoFrameFormat)param.format;
    digitalParam.width = param.size.width;
    digitalParam.height = param.size.height;
    digitalParam.rotation = param.rotation;

    for (int i = 0; i < 4; i++) {
        [digitalParam setStride: param.strides[i] atIndex:i];
    }

    // 遍历所有数字人API进行数据回调
    for (id<IZegoDigitalMobile> digitalMobile in self.digitalMobileArray) {
        [digitalMobile onRemoteVideoFrameRawData:data dataLength:dataLength param:digitalParam streamID:streamID];
    }
}

- (void)onPlayerSyncRecvSEI:(NSData *)data streamID:(NSString *)streamID{
    // 遍历所有数字人API进行SEI数据回调
    for (id<IZegoDigitalMobile> digitalMobile in self.digitalMobileArray) {
        [digitalMobile onPlayerSyncRecvSEI:streamID data:data];
    }
}

客户端登录房间并推流

// 记录智能体
self.streamToPlay = [self getAgentStreamID];

ZegoEngineConfig* engineConfig = [[ZegoEngineConfig alloc] init];
engineConfig.advancedConfig = @{
// !mark(1:5)
    @"set_audio_volume_ducking_mode":@1,/**该配置是用来做音量闪避的**/
    @"enable_rnd_volume_adaptive":@"true",/**该配置是用来做播放音量自适用**/
    //数字人
    @"sideinfo_callback_version":@(3),
    @"sideinfo_bound_to_video_decoder":@"true"
};
// !mark
[ZegoExpressEngine setEngineConfig:engineConfig];

//这个设置只影响AEC（回声消除），我们这里设置为ModeGeneral，是会走我们自研的回声消除，这比较可控，
//如果其他选项，可能会走系统的回声消除，这在iphone手机上效果可能会更好，但如果在一些android机上效果可能不好
[[ZegoExpressEngine sharedEngine] setAudioDeviceMode:ZegoAudioDeviceModeGeneral];

//开启 AI 回声消除
// !mark(1:5)
[[ZegoExpressEngine sharedEngine] enableAGC:TRUE];
[[ZegoExpressEngine sharedEngine] enableAEC:TRUE];
[[ZegoExpressEngine sharedEngine] setAECMode:ZegoAECModeAIBalanced];
[[ZegoExpressEngine sharedEngine] enableANS:TRUE];
[[ZegoExpressEngine sharedEngine] setANSMode:ZegoANSModeMedium];

// 登录房间
// !mark
[self loginRoom:^(int errorCode, NSDictionary *extendedData) {
    if (errorCode!=0) {
        NSString* errorMsg =[NSString stringWithFormat:@"进入语音房间失败:%d", errorCode];
        completion(NO, errorMsg);
        return;
    }

    //进房后开始推流
    [self startPushlishStream];
}];

客户端拉智能体的流

//监听房间流信息更新状态，拉取智能体流播放
- (void)onRoomStreamUpdate:(ZegoUpdateType)updateType
                streamList:(NSArray<ZegoStream *> *)streamList
              extendedData:(nullable NSDictionary *)extendedData
                    roomID:(NSString *)roomID{
    if (updateType == ZegoUpdateTypeAdd) {
        for (int i=0; i<streamList.count; i++) {
            ZegoStream* item = [streamList objectAtIndex:i];

// !mark
            [self startPlayStream:item.streamID];
        }
    } else if(updateType == ZegoUpdateTypeDelete) {
        for (int i=0; i<streamList.count; i++) {
            ZegoStream* item = [streamList objectAtIndex:i];
            [[ZegoExpressEngine sharedEngine] stopPlayingStream:item.streamID];
        }
    }
}

/**
 * 通知业务后台结束通话
 *
 * @param completion 完成回调，返回操作结果
 * @discussion 该方法会向服务器发送结束通话的请求，用于释放AI智能体实例
 */
- (void)doStopCallWithCompletion:(void (^)(NSInteger code, NSString *message, NSDictionary *data))completion {
    // 构建请求URL
// !mark
    NSString *url = [NSString stringWithFormat:@"%@/api/stop", self.currentBaseURL];
    NSURL *requestURL = [NSURL URLWithString:url];

    // 创建请求
    NSMutableURLRequest *request = [[NSMutableURLRequest alloc] initWithURL:requestURL];
    request.HTTPMethod = @"POST";

    // 设置请求头
    [request setValue:@"application/json" forHTTPHeaderField:@"Content-Type"];

    // 创建请求参数
    NSMutableDictionary *params = [NSMutableDictionary dictionary];
    NSData *jsonData = [NSJSONSerialization dataWithJSONObject:params options:0 error:nil];
    request.HTTPBody = jsonData;

    // 创建会话
    NSURLSession *session = [NSURLSession sharedSession];

    // 发送请求
    NSURLSessionDataTask *task = [session dataTaskWithRequest:request
                                           completionHandler:^(NSData * _Nullable data,
                                                            NSURLResponse * _Nullable response,
                                                            NSError * _Nullable error) {
        dispatch_async(dispatch_get_main_queue(), ^{
            if (error) {
                if (completion) {
                    completion(-1, @"网络请求失败", nil);
                }
                return;
            }

            NSHTTPURLResponse *httpUrlResponse = (NSHTTPURLResponse *)response;
            if (httpUrlResponse.statusCode != 200) {
                if (completion) {
                    completion(httpUrlResponse.statusCode,
                             [NSString stringWithFormat:@"服务器错误: %ld", (long)httpUrlResponse.statusCode],
                             nil);
                }
                return;
            }

            NSError *jsonError;
            NSDictionary *dict = [NSJSONSerialization JSONObjectWithData:data options:0 error:&jsonError];
            if (jsonError) {
                if (completion) {
                    completion(-2, @"解析响应数据失败", nil);
                }
                return;
            }

            // 解析响应数据
            NSInteger code = [dict[@"code"] integerValue];
            NSString *message = dict[@"message"];
            NSDictionary *responseData = dict[@"data"];

            if (completion) {
                completion(code, message, responseData);
            }

            // 退出房间
// !mark
            [[ZegoExpressEngine sharedEngine] logoutRoom];
        });
    }];

    [task resume];
}

ZegoEngineProfile* profile = [[ZegoEngineProfile alloc]init];
profile.appID = kZegoAppId;
profile.scenario = ZegoScenarioHighQualityChatroom; //高品质语聊房场景，设置该场景可以避免申请相机权限，接入方应按自己的业务场景设置具体值
ZegoEngineConfig* engineConfig = [[ZegoEngineConfig alloc] init];
engineConfig.advancedConfig = @{
    @"set_audio_volume_ducking_mode":@1,/**配置音量闪避，避免声音冲突**/
    @"enable_rnd_volume_adaptive":@"true",/**启用播放音量自适应**/
    @"sideinfo_callback_version":@(3),
    @"sideinfo_bound_to_video_decoder":@"true"
};
[ZegoExpressEngine setEngineConfig:engineConfig];
[ZegoExpressEngine createEngineWithProfile:profile eventHandler:self];
// 开启传统音频 3A 处理
[[ZegoExpressEngine sharedEngine] enableAGC:TRUE];
[[ZegoExpressEngine sharedEngine] enableAEC:TRUE];
[[ZegoExpressEngine sharedEngine] enableANS:TRUE];
// 开启 AI 回声消除
[[ZegoExpressEngine sharedEngine] setAECMode:ZegoAECModeAIBalanced];
// 开启 AI 降噪，适度的噪声抑制
[[ZegoExpressEngine sharedEngine] setANSMode:ZegoANSModeMedium];

// !mark
    [[ZegoExpressEngine sharedEngine] setPlayStreamBufferIntervalRange:streamId min:100 max:2000];

    [[ZegoExpressEngine sharedEngine] startPlayingStream:streamId];