284 lines
12 KiB
Swift
284 lines
12 KiB
Swift
//
|
||
// SpatialVideoConvertor.swift
|
||
// SwiftProject
|
||
//
|
||
// Created by aaa on 2024/3/7.
|
||
//
|
||
|
||
import Foundation
|
||
import AVKit
|
||
import VideoToolbox
|
||
import CoreImage
|
||
import ImageIO
|
||
|
||
class SpatialVideoConvertor {
|
||
|
||
///在立体视频中，正在处理的当前帧的左眼视图。
|
||
var leftEyeImage: CVPixelBuffer?
|
||
|
||
///在立体视频中，正在处理的当前帧的右眼视图。
|
||
var rightEyeImage: CVPixelBuffer?
|
||
|
||
|
||
|
||
|
||
func convertVideo( asset : AVAsset, outputFile: URL,type:Video3DFormat, progress: ((Float,Bool)->())? = nil ) async throws {
|
||
do {
|
||
try FileManager.default.removeItem(atPath: outputFile.path)
|
||
print("视频文件删除成功")
|
||
} catch {
|
||
print("删除视频文件出错：\(error)")
|
||
}
|
||
|
||
let assetReader = try AVAssetReader(asset: asset)
|
||
|
||
|
||
//检查是否为空间视频 print("该视频不是空间视频或图片")
|
||
let userDataItems = try await asset.loadMetadata(for:.quickTimeMetadata)
|
||
let spacialCharacteristics = userDataItems.filter { $0.identifier?.rawValue == "mdta/com.apple.quicktime.spatial.format-version" }
|
||
if spacialCharacteristics.count == 0 {
|
||
print("不是空间视频")
|
||
}
|
||
|
||
//获取输入视频的方向和大小(用于设置输出方向)
|
||
let (orientation, videoSize) = try await getOrientationAndResolutionSizeForVideo(asset: asset)
|
||
|
||
//输出宽度为宽度的一半
|
||
//我们有两个并排的视频，我们保持长宽比
|
||
var vw:VideoWriter?
|
||
|
||
|
||
|
||
//加载视频轨道
|
||
let output_video = try await AVAssetReaderTrackOutput(
|
||
track: asset.loadTracks(withMediaType: .video).first!,
|
||
outputSettings: [
|
||
AVVideoDecompressionPropertiesKey: [
|
||
kVTDecompressionPropertyKey_RequestedMVHEVCVideoLayerIDs: [0, 1] as CFArray,
|
||
],
|
||
]
|
||
)
|
||
assetReader.add(output_video)
|
||
//加载音频轨道
|
||
// let outputSettings_Audio:[String:Any] = [
|
||
// AVFormatIDKey:kAudioFormatLinearPCM,
|
||
// AVLinearPCMIsBigEndianKey:false,
|
||
// AVLinearPCMIsFloatKey:false,
|
||
// AVLinearPCMBitDepthKey:16
|
||
// ]
|
||
// let outputSettings_Audio = [
|
||
// AVFormatIDKey: kAudioFormatLinearPCM,
|
||
// AVSampleRateKey: 44100,
|
||
// AVNumberOfChannelsKey: 2,
|
||
//// AVEncoderBitRateKey: 128000,
|
||
// AVLinearPCMIsBigEndianKey:false,
|
||
// AVLinearPCMIsFloatKey:false,
|
||
// AVLinearPCMBitDepthKey:16
|
||
// ] as [String : Any]
|
||
|
||
let outputSettings_Audio = [
|
||
AVFormatIDKey: kAudioFormatLinearPCM, // 指定未压缩格式
|
||
AVSampleRateKey: 44100,
|
||
AVNumberOfChannelsKey: 2,
|
||
]
|
||
|
||
let output_audio = try await AVAssetReaderTrackOutput(
|
||
track: asset.loadTracks(withMediaType: .audio).first!,
|
||
outputSettings:outputSettings_Audio
|
||
)
|
||
if assetReader.canAdd(output_audio){
|
||
assetReader.add(output_audio)
|
||
print("添加音频read output成功。。。。")
|
||
}
|
||
else{
|
||
print("添加音频read output失败。。。。")
|
||
}
|
||
|
||
|
||
|
||
// let output_audio = AVAssetReaderAudioMixOutput(audioTracks: asset.tracks(withMediaType: .audio), audioSettings: nil)
|
||
// assetReader.add(output_audio)
|
||
|
||
|
||
|
||
assetReader.startReading()
|
||
|
||
let duration = try await asset.load(.duration)
|
||
|
||
|
||
while let nextSampleBuffer = output_video.copyNextSampleBuffer() {
|
||
|
||
guard let taggedBuffers = nextSampleBuffer.taggedBuffers else { return }
|
||
|
||
let leftEyeBuffer = taggedBuffers.first(where: {
|
||
$0.tags.first(matchingCategory: .stereoView) == .stereoView(.leftEye)
|
||
})?.buffer
|
||
let rightEyeBuffer = taggedBuffers.first(where: {
|
||
$0.tags.first(matchingCategory: .stereoView) == .stereoView(.rightEye)
|
||
})?.buffer
|
||
|
||
|
||
if let leftEyeBuffer,
|
||
let rightEyeBuffer,
|
||
case let .pixelBuffer(leftEyePixelBuffer) = leftEyeBuffer,
|
||
case let .pixelBuffer(rightEyePixelBuffer) = rightEyeBuffer {
|
||
|
||
leftEyeImage = leftEyePixelBuffer
|
||
rightEyeImage = rightEyePixelBuffer
|
||
|
||
let lciImage = CIImage(cvPixelBuffer: leftEyePixelBuffer)
|
||
let rciImage = CIImage(cvPixelBuffer: rightEyePixelBuffer)
|
||
|
||
let left = UIImage(ciImage: lciImage )
|
||
let right = UIImage(ciImage: rciImage )
|
||
|
||
var newpb:CIImage?
|
||
var cwidth:CGFloat
|
||
var cheight:CGFloat
|
||
switch type {
|
||
case .HSBS:
|
||
cwidth = left.size.width
|
||
cheight = left.size.height
|
||
newpb = joinImages_sbs(left: left, right: right, imgWidth: cwidth, imgHeight:cheight )
|
||
break
|
||
case .FSBS:
|
||
cwidth = left.size.width*2
|
||
cheight = left.size.height
|
||
newpb = joinImages_sbs(left: left, right: right, imgWidth: cwidth, imgHeight: cheight)
|
||
break
|
||
case .HOU:
|
||
cwidth = left.size.width
|
||
cheight = left.size.height
|
||
newpb = joinImages_ou(left: left, right: right, imgWidth: cwidth, imgHeight: cheight)
|
||
break
|
||
case .FOU:
|
||
cwidth = left.size.width
|
||
cheight = left.size.height*2
|
||
newpb = joinImages_ou(left: left, right: right, imgWidth: cwidth, imgHeight: cheight)
|
||
break
|
||
}
|
||
|
||
let time = CMSampleBufferGetOutputPresentationTimeStamp(nextSampleBuffer)
|
||
|
||
if vw == nil {
|
||
// vw = VideoWriter(url: outputFile, width: Int(cwidth), height: Int(cheight), orientation: orientation, sessionStartTime: CMTime(value: 1, timescale: 30 ), isRealTime: true, queue: .main)
|
||
vw = VideoWriter(url: outputFile, width: Int(cwidth), height: Int(cheight), orientation: orientation, sessionStartTime: CMTimeMake(value: 0, timescale: 1), isRealTime: true, queue: .main)
|
||
}
|
||
_ = vw!.add(image: newpb!, presentationTime: time)
|
||
print( "Added frame at \(time)")
|
||
|
||
// callback with progress
|
||
progress?( Float(time.value)/Float(duration.value),false)
|
||
|
||
// This sleep is needed to stop memory blooming - keeps around 280Mb rather than spiraling up to 8+Gig!
|
||
try await Task.sleep(nanoseconds: 3_000_000)
|
||
}
|
||
}
|
||
|
||
|
||
vw!.addAudio(assetTrackOutput: output_audio)
|
||
|
||
|
||
print( "status - \(assetReader.status)")
|
||
print( "status - \(assetReader.error?.localizedDescription ?? "None")")
|
||
print( "Finished")
|
||
|
||
_ = try await vw!.finish()
|
||
progress?( Float(1.0),true)
|
||
|
||
}
|
||
|
||
|
||
|
||
|
||
//获取ciimage的数据
|
||
func isSpatialImage2(from ciImage: CIImage) {
|
||
let context = CIContext()
|
||
guard let cgImage = context.createCGImage(ciImage, from: ciImage.extent) else {
|
||
return
|
||
}
|
||
let dataProvider = CGDataProvider(data: cgImage.dataProvider!.data! as CFData)
|
||
let imageSource = CGImageSourceCreateWithDataProvider(dataProvider!, nil)
|
||
let frameCount = CGImageSourceGetCount(imageSource!)
|
||
print(frameCount)
|
||
for index in 0..<frameCount {
|
||
let properties = CGImageSourceCopyPropertiesAtIndex(imageSource!, index, nil) as? [CFString: Any]
|
||
print(properties as Any)
|
||
|
||
guard let frameImage = CGImageSourceCreateImageAtIndex(imageSource!, index, nil) else {
|
||
continue
|
||
}
|
||
print(frameImage)
|
||
}
|
||
}
|
||
|
||
func getOrientationAndResolutionSizeForVideo(asset:AVAsset) async throws -> (CGAffineTransform, CGSize) {
|
||
guard let track = try await asset.loadTracks(withMediaType: AVMediaType.video).first
|
||
else{throw VideoReaderError.invalidVideo}
|
||
let naturalSize = try await track.load(.naturalSize)
|
||
let naturalTransform = try await track.load(.preferredTransform)
|
||
let size = naturalSize.applying(naturalTransform)
|
||
return (naturalTransform, CGSize(width: abs(size.width), height: abs(size.height)) )
|
||
}
|
||
|
||
//将两张图片合成一张图片
|
||
func joinImages( leftImage:CIImage, rightImage:CIImage) -> CIImage {
|
||
let left = UIImage(ciImage: leftImage )
|
||
let right = UIImage(ciImage: rightImage )
|
||
|
||
let imageWidth = left.size.width/2 + right.size.width/2
|
||
let imageHeight = left.size.height/2
|
||
|
||
let newImageSize = CGSize(width:imageWidth, height: imageHeight);
|
||
UIGraphicsBeginImageContextWithOptions(newImageSize, false, 1);
|
||
left.draw(in: CGRect(x:0, y:0, width:imageWidth/2, height:imageHeight))
|
||
right.draw(in: CGRect(x:imageWidth/2, y:0, width:imageWidth/2, height:imageHeight))
|
||
let image = UIGraphicsGetImageFromCurrentImageContext()!
|
||
UIGraphicsEndImageContext();
|
||
|
||
let ci = CIImage(cgImage: image.cgImage!)
|
||
return ci
|
||
}
|
||
|
||
|
||
|
||
|
||
//将两张图片合成一张图片 SBS
|
||
func joinImages_sbs( left:UIImage, right:UIImage,imgWidth:CGFloat,imgHeight:CGFloat) -> CIImage {
|
||
let newImageSize = CGSize(width:imgWidth, height: imgHeight);
|
||
UIGraphicsBeginImageContextWithOptions(newImageSize, false, 1);
|
||
left.draw(in: CGRect(x:0, y:0, width:imgWidth/2, height:imgHeight))
|
||
right.draw(in: CGRect(x:imgWidth/2, y:0, width:imgWidth/2, height:imgHeight))
|
||
let image = UIGraphicsGetImageFromCurrentImageContext()!
|
||
UIGraphicsEndImageContext();
|
||
|
||
// DispatchQueue.main.async {
|
||
// var iv:UIImageView? = KWindow?.viewWithTag(9988) as? UIImageView
|
||
// if let imgv = iv {
|
||
// imgv.image = image
|
||
// }
|
||
// else {
|
||
// iv = UIImageView(frame: CGRect(origin: CGPoint(x: 0, y: 400), size: CGSize(width: KScreenWidth, height: image.size.height*KScreenWidth/image.size.width)))
|
||
// iv!.tag = 9988
|
||
// KWindow?.addSubview(iv!)
|
||
// }
|
||
// }
|
||
|
||
let ci = CIImage(cgImage: image.cgImage!)
|
||
return ci
|
||
}
|
||
|
||
//将两张图片合成一张图片 OU
|
||
func joinImages_ou( left:UIImage, right:UIImage,imgWidth:CGFloat,imgHeight:CGFloat) -> CIImage {
|
||
let newImageSize = CGSize(width:imgWidth, height: imgHeight);
|
||
UIGraphicsBeginImageContextWithOptions(newImageSize, false, 1);
|
||
left.draw(in: CGRect(x:0, y:0, width:imgWidth, height:imgHeight/2))
|
||
right.draw(in: CGRect(x:0, y:imgHeight/2, width:imgWidth, height:imgHeight/2))
|
||
let image = UIGraphicsGetImageFromCurrentImageContext()!
|
||
UIGraphicsEndImageContext();
|
||
|
||
let ci = CIImage(cgImage: image.cgImage!)
|
||
return ci
|
||
}
|
||
}
|