Composing Video and Audio - Video's audio is gone

Question

My question is, I am using the function below, to compose a video and audio. I want to keep video\'s original sound but it goes away somehow, I do not have any clue.

I

Answer 1

Here's the combined code of all the answers. It took me a while to decode those answers so I decided to add it here for any future users:

Swift 5:

enum MixError: Error {
   case TimeRangeFailure
   case ExportFailure
}

var selectedVideoLevel = 1.0
var selectedMusicLevel = 1.0

func mix(videoUrl: URL, musicUrl: URL, completion: ((Result<URL, Error>) -> Void)?) {
    let videoAsset = AVAsset(url: videoUrl)
    let musicAsset = AVAsset(url: musicUrl)

    let audioVideoComposition = AVMutableComposition()

    let audioMix = AVMutableAudioMix()
    var mixParameters = [AVMutableAudioMixInputParameters]()

    let videoCompositionTrack = audioVideoComposition
      .addMutableTrack(withMediaType: .video, preferredTrackID: .init())!

    let audioCompositionTrack = audioVideoComposition
      .addMutableTrack(withMediaType: .audio, preferredTrackID: .init())!

    let musicCompositionTrack = audioVideoComposition
      .addMutableTrack(withMediaType: .audio, preferredTrackID: .init())!

    let videoAssetTrack = videoAsset.tracks(withMediaType: .video)[0]
    let audioAssetTrack = videoAsset.tracks(withMediaType: .audio).first
    let musicAssetTrack = musicAsset.tracks(withMediaType: .audio)[0]

    let audioParameters = AVMutableAudioMixInputParameters(track: audioAssetTrack)
    audioParameters.trackID = audioCompositionTrack.trackID

    let musicParameters = AVMutableAudioMixInputParameters(track: musicAssetTrack)
    musicParameters.trackID = musicCompositionTrack.trackID

    audioParameters.setVolume(selectedVideoLevel, at: .zero)
    musicParameters.setVolume(selectedMusicLevel, at: .zero)

    mixParameters.append(audioParameters)
    mixParameters.append(musicParameters)

    audioMix.inputParameters = mixParameters

    /// prevents video from unnecessary rotations
    videoCompositionTrack.preferredTransform = videoAssetTrack.preferredTransform

    do {
      let timeRange = CMTimeRange(start: .zero, duration: videoAsset.duration)

      try videoCompositionTrack.insertTimeRange(timeRange, of: videoAssetTrack, at: .zero)

      if let audioAssetTrack = audioAssetTrack {
        try audioCompositionTrack.insertTimeRange(timeRange, of: audioAssetTrack, at: .zero)
      }

      try musicCompositionTrack.insertTimeRange(timeRange, of: musicAssetTrack, at: .zero)

    } catch {
      completion?(.failure(MixError.TimeRangeFailure)
    }


    let exportUrl = FileManager.default
      .urls(for: .applicationSupportDirectory, in: .userDomainMask).first?
      .appendingPathComponent("\(Date().timeIntervalSince1970)-video.mp4")

    let exportSession = AVAssetExportSession(
      asset: audioVideoComposition,
      presetName: AVAssetExportPresetHighestQuality
    )

    exportSession?.audioMix = audioMix
    exportSession?.outputFileType = .m4v
    exportSession?.outputURL = exportUrl

    exportSession?.exportAsynchronously(completionHandler: {
      guard let status = exportSession?.status else { return }

      switch status {
      case .completed:
        completion?(.success(exportUrl!))
     case .failed:
        completion?(.failure(MixError.ExportError)
      default:
        print(status)

      }

    })

  }

Answer 2

I figured it out. It seems an AVAsset which loads a video holds the audio and video separately. So you can reach them writing``

videoAsset.tracks(withMediaType: AVMediaTypeAudio)[0] //audio of a video
videoAsset.tracks(withMediaType: AVMediaTypeVideo)[0] //video of a video(without sound)

So I added these lines to the code and it worked!

var mutableCompositionBackTrack : [AVMutableCompositionTrack] = []

mutableCompositionBackTrack.append(mixComposition.addMutableTrack(withMediaType: AVMediaTypeAudio, preferredTrackID: kCMPersistentTrackID_Invalid)) 

try mutableCompositionBackTrack[0].insertTimeRange(CMTimeRangeMake(kCMTimeZero, videoAssetTrack.timeRange.duration), of: backAssetTrack, at: kCMTimeZero)

There is a still missing point that I do not know how to do, and that is setting volumes of these audio assets. I will update this answer as soon as I figure out how.

Answer 3

You can adjust volume for video and audio separately @Faruk, Here a is little bit code for that.

        //Extract audio from the video and the music
let audioMix: AVMutableAudioMix = AVMutableAudioMix()
var audioMixParam: [AVMutableAudioMixInputParameters] = []

let assetVideoTrack: AVAssetTrack = assetVideo.tracksWithMediaType(AVMediaTypeAudio)[0]
let assetMusicTrack: AVAssetTrack = assetMusic.tracksWithMediaType(AVMediaTypeAudio)[0]

let videoParam: AVMutableAudioMixInputParameters = AVMutableAudioMixInputParameters(track: assetVideoTrack)
videoParam.trackID = compositionAudioVideo.trackID

let musicParam: AVMutableAudioMixInputParameters = AVMutableAudioMixInputParameters(track: assetMusicTrack)
musicParam.trackID = compositionAudioMusic.trackID

//Set final volume of the audio record and the music
videoParam.setVolume(volumeVideo, atTime: kCMTimeZero)
musicParam.setVolume(volumeAudio, atTime: kCMTimeZero)

//Add setting
audioMixParam.append(musicParam)
audioMixParam.append(videoParam)

//Add audio on final record
//First: the audio of the record and Second: the music
do {
try compositionAudioVideo.insertTimeRange(CMTimeRangeMake(kCMTimeZero, assetVideo.duration), ofTrack: assetVideoTrack, atTime: kCMTimeZero)
} catch _ {
assertionFailure()
}

do {
try compositionAudioMusic.insertTimeRange(CMTimeRangeMake(CMTimeMake(Int64(startAudioTime * 10000), 10000), assetVideo.duration), ofTrack: assetMusicTrack, atTime: kCMTimeZero)
} catch _ {
assertionFailure()
}

//Add parameter
audioMix.inputParameters = audioMixParam

let completeMovie = "\(docsDir)/\(randomString(5)).mp4"
let completeMovieUrl = NSURL(fileURLWithPath: completeMovie)
let exporter: AVAssetExportSession = AVAssetExportSession(asset: composition, presetName: AVAssetExportPresetHighestQuality)!

exporter.outputURL = completeMovieUrl
exporter.outputFileType = AVFileTypeMPEG4
exporter.audioMix = audioMix
exporter.exportAsynchronouslyWithCompletionHandler({ 

switch exporter.status {

case AVAssetExportSessionStatus.Completed:
    print("success with output url \(completeMovieUrl)")
    case  AVAssetExportSessionStatus.Failed:
        print("failed \(String(exporter.error))")
    case AVAssetExportSessionStatus.Cancelled:
        print("cancelled \(String(exporter.error))")
    default:
        print("complete")
    }            
})

}