422 lines
17 KiB
Swift
422 lines
17 KiB
Swift
//
|
||
// SpatialVideoConverter.swift
|
||
// vp
|
||
//
|
||
// Created by soldoros on 2024/1/17.
|
||
//
|
||
|
||
import AVFoundation
|
||
import CoreImage
|
||
import Foundation
|
||
import Observation
|
||
import VideoToolbox
|
||
|
||
///将立体视频转换为空间视频的处理器。
|
||
class SpatialVideoConverter:NSObject {
|
||
// MARK: - Properties
|
||
|
||
// MARK: Public
|
||
|
||
///需要从源视频中处理的帧总数。
|
||
var totalFrames: Double = .zero
|
||
|
||
///到目前为止已经处理的帧数。
|
||
var framesProcessed: Double = 0.0
|
||
|
||
///转换完成前的剩余时间。
|
||
var timeRemaining: Double = .zero
|
||
|
||
///处理开始的时间戳。
|
||
var startTime: Date = .now
|
||
|
||
///一个布尔标志，确定处理器是否已经在使用中。
|
||
var isProcessing = false
|
||
|
||
///最后一个成功转换的文件。
|
||
var lastConvertedFileURL: LastConvertedFile?
|
||
|
||
///在立体视频中，正在处理的当前帧的左眼视图。
|
||
var leftEyeImage: CVPixelBuffer?
|
||
|
||
///在立体视频中，正在处理的当前帧的右眼视图。
|
||
var rightEyeImage: CVPixelBuffer?
|
||
|
||
// MARK: Private
|
||
|
||
///一个辅助处理器，可以用来裁剪和转换底层图像类型。
|
||
private let processor = FrameProcessor()
|
||
|
||
///将视频帧写入文件的视频写入器。
|
||
private var writer: AVAssetWriter?
|
||
|
||
///分配avassetwwriter的视频输入的队列。
|
||
private let videoInputQueue = DispatchQueue(label: "com.test.spatialWriterVideo")
|
||
|
||
///分配avassetwwriter的音频输入的队列。
|
||
private let audioInputQueue = DispatchQueue(label: "com.test.spatialWriterAudio")
|
||
|
||
///“avassetwwriter”的输入接收视频帧。
|
||
private var writerVideoInput: AVAssetWriterInput?
|
||
|
||
///“avassetwwriter”的输入接收音频样本。
|
||
private var writerAudioInput: AVAssetWriterInput?
|
||
|
||
///处理源视频的读取器。
|
||
private var heroReader: AVAssetReader?
|
||
|
||
/// AVAssetReader的视频轨迹输出
|
||
private var readerVideoOutput: AVAssetReaderTrackOutput?
|
||
|
||
/// AVAssetReader的音轨输出。
|
||
private var readerAudioOutput: AVAssetReaderTrackOutput?
|
||
|
||
///一个内部标志，指示视频写入器是否完成所有帧的处理。
|
||
private var videoWritingFinished = false
|
||
|
||
///一个内部标志，指示音频写入器是否完成所有帧的处理。
|
||
private var audioWritingFinished = false
|
||
|
||
///一个格式化程序，可用于将时间戳(如处理时间)转换为字符串。
|
||
private var dateFormatter: DateComponentsFormatter {
|
||
let formatter = DateComponentsFormatter()
|
||
formatter.allowedUnits = [.day, .hour, .minute, .second]
|
||
formatter.unitsStyle = .abbreviated
|
||
return formatter
|
||
}
|
||
|
||
///一个格式化程序，可用于将字节计数(如文件大小)转换为字符串。
|
||
private var byteCountFormatter: ByteCountFormatter {
|
||
let formatter = ByteCountFormatter()
|
||
formatter.allowedUnits = [.useGB, .useMB, .useKB]
|
||
formatter.countStyle = .file
|
||
return formatter
|
||
}
|
||
|
||
|
||
// MARK: - Methods
|
||
|
||
// MARK: Public
|
||
|
||
///将源立体视频转换为空间视频。
|
||
/// -参数:
|
||
/// - sourceVideoURL:源立体视频的URL。必须是AVFoundation支持的格式，如H.264、H.265或ProRes文件。
|
||
/// - outputVideoURL:如果转换成功完成，输出视频的URL。
|
||
func convertStereoscopicVideoToSpatialVideo(
|
||
sourceVideoURL: URL,
|
||
outputVideoURL: URL,
|
||
progress: ((Float)->())? = nil
|
||
) async throws {
|
||
|
||
let heroAsset = AVAsset(url: sourceVideoURL)
|
||
|
||
//删除临时文件，如果它存在。
|
||
try removeExistingFile(at: outputVideoURL)
|
||
|
||
writer = try AVAssetWriter(outputURL: outputVideoURL, fileType: .mov)
|
||
guard let videoTrack = try await heroAsset.loadTracks(withMediaType: .video).first else {
|
||
return
|
||
}
|
||
let audioTrack = try await heroAsset.loadTracks(withMediaType: .audio).first
|
||
|
||
guard let videoFormatDescription = try await videoTrack.load(.formatDescriptions).first else {
|
||
return
|
||
}
|
||
|
||
if !processor.isPrepared {
|
||
processor.prepare(with: videoFormatDescription, outputRetainedBufferCountHint: 1)
|
||
}
|
||
|
||
//设置源视频大小并计算左眼和右眼图像的区域。
|
||
let naturalSize = try await videoTrack.load(.naturalSize)
|
||
let leftEyeRegion = CGRect(
|
||
x: 0,
|
||
y: 0,
|
||
width: naturalSize.width / 2,
|
||
height: naturalSize.height
|
||
)
|
||
let rightEyeRegion = CGRect(
|
||
x: naturalSize.width / 2,
|
||
y: 0,
|
||
width: naturalSize.width / 2,
|
||
height: naturalSize.height
|
||
)
|
||
|
||
//配置帧率和帧数，提供给视图计算剩余时间。
|
||
let frameRate = try await videoTrack.load(.nominalFrameRate)
|
||
let dataRate = try await videoTrack.load(.estimatedDataRate)
|
||
let duration = try await heroAsset.load(.duration)
|
||
let frames = CMTimeGetSeconds(duration) * Double(frameRate)
|
||
totalFrames = frames
|
||
|
||
// TODO:添加功能来指定所需的输出分辨率，比特率和
|
||
// TODO:水平视差。
|
||
|
||
//设置视频输出设置
|
||
var videoSettings = AVOutputSettingsAssistant(preset: .mvhevc1440x1440)?.videoSettings
|
||
videoSettings?[AVVideoWidthKey] = leftEyeRegion.width
|
||
videoSettings?[AVVideoHeightKey] = leftEyeRegion.height
|
||
var compressionProperties = videoSettings?[AVVideoCompressionPropertiesKey] as! [String: Any]
|
||
compressionProperties[AVVideoAverageBitRateKey] = dataRate
|
||
compressionProperties[kVTCompressionPropertyKey_HorizontalDisparityAdjustment as String] = 0
|
||
compressionProperties[kCMFormatDescriptionExtension_HorizontalFieldOfView as String] = 90
|
||
videoSettings?[AVVideoCompressionPropertiesKey] = compressionProperties
|
||
|
||
//设置视频输入设置
|
||
writerVideoInput = AVAssetWriterInput(mediaType: .video, outputSettings: videoSettings)
|
||
guard let writerVideoInput else { return }
|
||
writerVideoInput.expectsMediaDataInRealTime = false
|
||
|
||
//设置音频输入设置，如果有音频轨道
|
||
if audioTrack != nil {
|
||
writerAudioInput = AVAssetWriterInput(mediaType: .audio, outputSettings: nil)
|
||
}
|
||
|
||
//设置像素缓冲组适配器，用于写左右眼框。
|
||
let pixelBufferAdaptor = AVAssetWriterInputTaggedPixelBufferGroupAdaptor(
|
||
assetWriterInput: writerVideoInput,
|
||
sourcePixelBufferAttributes: .none)
|
||
|
||
//设置阅读器的立体视频文件，或左眼(英雄眼)视图，如果视图提供给每一个
|
||
//个人眼睛。
|
||
let readerOutputSettings: [String:Any] = [
|
||
kCVPixelBufferPixelFormatTypeKey as String: NSNumber(value: kCVPixelFormatType_32BGRA),
|
||
kCVPixelBufferWidthKey as String: naturalSize.width,
|
||
kCVPixelBufferHeightKey as String: naturalSize.height
|
||
]
|
||
readerVideoOutput = AVAssetReaderTrackOutput(
|
||
track: videoTrack,
|
||
outputSettings: readerOutputSettings)
|
||
if let audioTrack {
|
||
readerAudioOutput = AVAssetReaderTrackOutput(track: audioTrack, outputSettings: nil)
|
||
}
|
||
heroReader = try AVAssetReader(asset: heroAsset)
|
||
|
||
//向读取器添加必要的输出并开始读取。
|
||
guard let heroReader,
|
||
let readerVideoOutput
|
||
else { return }
|
||
heroReader.add(readerVideoOutput)
|
||
if let readerAudioOutput {
|
||
heroReader.add(readerAudioOutput)
|
||
}
|
||
heroReader.startReading()
|
||
|
||
//向写入器添加必要的输入并启动会话。
|
||
guard let writer else { return }
|
||
writer.add(writerVideoInput)
|
||
if let writerAudioInput {
|
||
writer.add(writerAudioInput)
|
||
}
|
||
writer.startWriting()
|
||
writer.startSession(atSourceTime: .zero)
|
||
|
||
//表示我们已经开始处理。
|
||
isProcessing = true
|
||
|
||
//设置处理的开始时间，允许下游估计时间
|
||
//完成计算。
|
||
startTime = Date.now
|
||
|
||
writerVideoInput.requestMediaDataWhenReady(on: videoInputQueue) { [weak self] in
|
||
guard let self else { return }
|
||
while writerVideoInput.isReadyForMoreMediaData {
|
||
autoreleasepool {
|
||
guard self.processor.isPrepared else {
|
||
print("The processor is not prepared. Cannot write video")
|
||
return
|
||
}
|
||
|
||
if let frame = readerVideoOutput.copyNextSampleBuffer(),
|
||
let frameBuffer = CMSampleBufferGetImageBuffer(frame) {
|
||
let sourceBuffer = CIImage(cvImageBuffer: frameBuffer)
|
||
|
||
// Setup the left and right eye `CVPixelBuffer` references.
|
||
guard let leftEye = self.processor.cropPixelBuffer(
|
||
pixelBufferImage: sourceBuffer,
|
||
targetRect: leftEyeRegion
|
||
),
|
||
let rightEye = self.processor.cropPixelBuffer(
|
||
pixelBufferImage: sourceBuffer,
|
||
targetRect: rightEyeRegion
|
||
)
|
||
else { return }
|
||
|
||
// Set a video preview
|
||
self.leftEyeImage = leftEye
|
||
self.rightEyeImage = rightEye
|
||
|
||
// Create an array of `CMTaggedBuffers, one for each eye's view.
|
||
let taggedBuffers: [CMTaggedBuffer] = [
|
||
.init(tags: [.videoLayerID(0), .stereoView(.leftEye)], pixelBuffer: leftEye),
|
||
.init(tags: [.videoLayerID(1), .stereoView(.rightEye)], pixelBuffer: rightEye)
|
||
]
|
||
|
||
let didAppend = pixelBufferAdaptor.appendTaggedBuffers(
|
||
taggedBuffers,
|
||
withPresentationTime: frame.presentationTimeStamp
|
||
)
|
||
if !didAppend {
|
||
print("Failed to append frame.")
|
||
}
|
||
|
||
// Increment the number of frames processed.
|
||
if self.framesProcessed < (self.totalFrames - 1) {
|
||
self.framesProcessed += 1
|
||
}
|
||
|
||
// Calculate the estimated time remaining based on how long this frame took to process.
|
||
self.calculateTimeRemaining()
|
||
progress?( Float(self.framesProcessed)/Float(self.totalFrames))
|
||
|
||
} else {
|
||
if !self.videoWritingFinished {
|
||
sourceVideoURL.stopAccessingSecurityScopedResource()
|
||
self.videoWritingFinished.toggle()
|
||
writerVideoInput.markAsFinished()
|
||
self.stop(with: outputVideoURL)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
//当它准备好时通知编写器的音频输入请求数据。
|
||
if let writerAudioInput,
|
||
let readerAudioOutput {
|
||
writerAudioInput.requestMediaDataWhenReady(on: audioInputQueue) { [weak self] in
|
||
guard let self else { return }
|
||
while writerAudioInput.isReadyForMoreMediaData {
|
||
autoreleasepool {
|
||
if let sample = readerAudioOutput.copyNextSampleBuffer() {
|
||
writerAudioInput.append(sample)
|
||
} else {
|
||
if !self.audioWritingFinished {
|
||
self.audioWritingFinished.toggle()
|
||
writerAudioInput.markAsFinished()
|
||
self.stop(with: outputVideoURL)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
|
||
///取消当前正在转换的视频，并将写入器重置到准备转换的状态
|
||
///接受一个新文件进行转换。
|
||
/// -参数expectedOutputURL:被取消的文件的预期输出' URL '，因此
|
||
///可以删除必要的临时文件并重置权限。
|
||
func cancel(expectedOutputURL: URL) {
|
||
writerVideoInput?.markAsFinished()
|
||
writerAudioInput?.markAsFinished()
|
||
writer?.cancelWriting()
|
||
try? removeExistingFile(at: expectedOutputURL)
|
||
resetWriter()
|
||
}
|
||
|
||
// MARK: - Private
|
||
|
||
///删除输出URL中已经存在的文件。
|
||
/// -参数outputVideoURL:提供的视频输出URL。
|
||
private func removeExistingFile(at outputVideoURL: URL) throws {
|
||
try FileManager.default.removeItem(atPath: outputVideoURL.path)
|
||
}
|
||
|
||
///计算估计的处理剩余时间。
|
||
private func calculateTimeRemaining() {
|
||
let totalTimeElapsed = Date.now.timeIntervalSince1970 - startTime.timeIntervalSince1970
|
||
let totalFramesCompleted = framesProcessed
|
||
let averageTimeBetweenFrames = totalTimeElapsed / totalFramesCompleted
|
||
let estimatedTimeRemaining = averageTimeBetweenFrames * (totalFrames - totalFramesCompleted)
|
||
guard self.timeRemaining != 0 else {
|
||
self.timeRemaining = estimatedTimeRemaining
|
||
return
|
||
}
|
||
if estimatedTimeRemaining < self.timeRemaining + 100 {
|
||
self.timeRemaining = estimatedTimeRemaining
|
||
}
|
||
}
|
||
|
||
///将输入标记为已完成，结束视频写入会话并完成写入。
|
||
///—参数outputURL:转换后文件的输出URL。
|
||
private func stop(with outputURL: URL) {
|
||
guard isProcessing,
|
||
let writerVideoInput,
|
||
videoWritingFinished
|
||
else { return }
|
||
|
||
if let writerAudioInput {
|
||
guard audioWritingFinished else { return }
|
||
}
|
||
|
||
self.writer?.finishWriting { [weak self] in
|
||
guard let self else {return}
|
||
Task {
|
||
try? await self.saveLastConvertedFile(outputURL: outputURL)
|
||
outputURL.stopAccessingSecurityScopedResource()
|
||
self.resetWriter()
|
||
}
|
||
|
||
print("finished writing")
|
||
}
|
||
}
|
||
|
||
///将读取、写入和时间计算重置到原始状态，为下一次计算做准备
|
||
///文件转换。
|
||
private func resetWriter() {
|
||
isProcessing = false
|
||
self.totalFrames = 0
|
||
self.framesProcessed = 0
|
||
self.timeRemaining = 0
|
||
self.startTime = .now
|
||
|
||
self.writerVideoInput = nil
|
||
self.writerAudioInput = nil
|
||
self.writer = nil
|
||
self.readerVideoOutput = nil
|
||
self.readerAudioOutput = nil
|
||
self.heroReader = nil
|
||
|
||
self.videoWritingFinished = false
|
||
self.audioWritingFinished = false
|
||
}
|
||
|
||
///创建一个引用到最后一个成功完成的文件进行处理，这个引用可以用于
|
||
///显示预览的用户访问最后转换的文件。
|
||
/// -参数outputURL:最后转换文件的输出' URL '。
|
||
private func saveLastConvertedFile(outputURL: URL) async throws {
|
||
do {
|
||
let attr = try FileManager.default.attributesOfItem(atPath: outputURL.path)
|
||
let fileSize = attr[FileAttributeKey.size] as? Int64
|
||
|
||
let asset = AVAsset(url: outputURL)
|
||
let duration = try await asset.load(.duration)
|
||
|
||
self.lastConvertedFileURL = LastConvertedFile(
|
||
filePath: outputURL,
|
||
timeToProcess: dateFormatter.string(from: startTime, to: Date.now) ?? "Unknown",
|
||
fileSize: byteCountFormatter.string(fromByteCount: fileSize ?? 0),
|
||
duration: dateFormatter.string(from: duration.seconds) ?? "Unknown"
|
||
)
|
||
} catch {
|
||
print("Error: \(error)")
|
||
}
|
||
}
|
||
}
|
||
|
||
///提供上次转换的视频文件的基本元数据的结构体。
|
||
struct LastConvertedFile: Codable, Equatable {
|
||
///最后一次转换的视频文件路径。
|
||
let filePath: URL
|
||
|
||
///将最后一个转换文件转换为字符串所花费的时间。
|
||
let timeToProcess: String
|
||
|
||
///最后转换文件的文件大小，字符串形式。
|
||
let fileSize: String
|
||
|
||
///最后转换文件的视频时长，字符串形式。
|
||
let duration: String
|
||
}
|