Golang : Merge video(OpenCV) and audio(PortAudio) into a mp4 file

The previous Golang tutorials cover on how to capture image frames from a web camera with OpenCV and record voice from microphone with PortAudio. For this tutorial, we will learn how to merge video stream and audio stream into a single file with the help of ffmpeg.

In the code example that follows, we will launch a goroutine to capture audio stream from microphone and simultaneously capture video stream from camera. The audio and video data will be encoded separately into two files and merge into a singe mp4 file with ffmpeg. AFAIK, there is no single stream way of capturing both audio and video from a web camera yet in Golang. This is the best solution that I can come up with. It is based on a similar program but written in Python - https://github.com/JRodrigoF/AVrecordeR/blob/master/AVrecordeR.py

Make sure you have OpenCV and PortAudio installed on your computer before building the code below.

Here you go!

---

 package main

 /*
  #include <stdio.h>
  #include <unistd.h>
  #include <termios.h>
  char getch(){
 char ch = 0;
 struct termios old = {0};
 fflush(stdout);
 if( tcgetattr(0, &old) < 0 ) perror("tcsetattr()");
 old.c_lflag &= ~ICANON;
 old.c_lflag &= ~ECHO;
 old.c_cc[VMIN] = 1;
 old.c_cc[VTIME] = 0;
 if( tcsetattr(0, TCSANOW, &old) < 0 ) perror("tcsetattr ICANON");
 if( read(0, &ch,1) < 0 ) perror("read()");
 old.c_lflag |= ICANON;
 old.c_lflag |= ECHO;
 if(tcsetattr(0, TCSADRAIN, &old) < 0) perror("tcsetattr ~ICANON");
 return ch;
  }
 */
 import "C"

 // stackoverflow.com/questions/14094190/golang-function-similar-to-getchar

 import (
 "fmt"
 "github.com/gordonklaus/portaudio"
 "github.com/lazywei/go-opencv/opencv"
 wave "github.com/zenwerk/go-wave"
 "math"
 "math/rand"
 "os"
 "os/exec"
 "path"
 "runtime"
 "time"
 )

 //global variables

 var (
 stopAudio = false
 stopVideo = false

 win = new(opencv.Window)
 waveWriter = new(wave.Writer)
 frameCounter = 1
 startTime = time.Now()
 )

 // recording in progress ticker. From good old DOS days.
 var ticker = []string{
 "-",
 "\\",
 "/",
 "|",
 }

 func errCheck(err error) {

 if err != nil {
 panic(err)
 }
 }

 func videoFeed(videoFileName string) {

 // activate webCamera
 webCamera := opencv.NewCameraCapture(opencv.CV_CAP_ANY) // autodetect

 if webCamera == nil {
 panic("Unable to open camera")
 }

 // !! NEED TO CHECK IF YOUR OS HAS THE CODECS INSTALLED BEFORE SELECTING THE CODEC !!
 // !! OTHERWISE, YOU WILL GET A VERY SMALL & CORRUPT VIDEO FILE !!

 // see http://www.fourcc.org/codecs.php for other possible combinations
 // opencv.FOURCC('p', 'i', 'm', '1') // MPEG-1 codec
 // opencv.FOURCC('m', 'j', 'p', 'g') // motion-jpeg codec
 // opencv.FOURCC('m', 'p', '4', '2') // MPEG-4.2 codec
 // opencv.FOURCC('d', 'i', 'v', '3') // MPEG-4.3 codec
 // opencv.FOURCC('m', 'p', '4', 'v') // MPEG-4 codec
 // opencv.FOURCC('u', '2', '6', '3') // H263 codec
 // opencv.FOURCC('i', '2', '6', '3') // H263I codec
 // opencv.FOURCC('f', 'l', 'v', '1') // FLV1 codec

 // codec := opencv.CV_FOURCC_PROMPT // Windows only. Prompt for codec selection
 // codec := int(webCamera.GetProperty(opencv.CV_CAP_PROP_FOURCC)) -- won't work on my Mac

 codec := int(opencv.FOURCC('m', 'p', '4', 'v')) // must be lower case, upper case will screw the file...

 fps := float32(25) // 25 frames per second
 //fps := float32(webCamera.GetProperty(opencv.CV_CAP_PROP_POS_FRAMES))
 frameWidth := int(webCamera.GetProperty(opencv.CV_CAP_PROP_FRAME_WIDTH))
 frameHeight := int(webCamera.GetProperty(opencv.CV_CAP_PROP_FRAME_HEIGHT))
 isColor := 1 // 0 = false(grayscale), 1 = true -- for Windows only I think

 // !! IMPORTANT : Remember to set the type to frameWidth and frameHeight for
 // for both input(src) and output(destination) as the same
 // otherwise, you gonna get this error message -
 // [OpenCV Error: Assertion failed (dst.data == dst0.data) in cvCvtColor,]

 // Just a note, you still can resize the frames before writing to file if you want

 // for more info, read http://docs.opencv.org/trunk/dd/d9e/classcv_1_1VideoWriter.html
 videoFileWriter := opencv.NewVideoWriter(videoFileName, codec, fps, frameWidth, frameHeight, isColor)

 win = opencv.NewWindow("Go-OpenCV recording video and audio to file")

 for {

 if !stopVideo { // to prevent segmentation fault

 if webCamera.GrabFrame() {
 imgFrame := webCamera.RetrieveFrame(1)

 if imgFrame != nil {
 win.ShowImage(imgFrame)

 // save frame to video file
 n := videoFileWriter.WriteFrame(imgFrame)
 frameCounter += n

 // 0.16 delay = 6 fps
 //time.Sleep(time.Duration(16/100) * time.Second)

 }
 }
 } else {
 break
 }
 }
 webCamera.Release()

 }

 func audioFeed(audioFileName string) {

 // for Audio
 waveFile, err := os.Create(audioFileName)
 errCheck(err)

 framesPerBuffer := make([]byte, 1024)

 // init PortAudio
 portaudio.Initialize()

 //use our default input parameters
 inputDevice, err := portaudio.DefaultInputDevice()
 errCheck(err)

 fmt.Println("Using [", inputDevice.Name, "] to record.")

 inputChannels := inputDevice.MaxInputChannels
 outputChannels := inputDevice.MaxOutputChannels
 // must match your microphone's sample rate - otherwise you will get Input Overflowed error
 sampleRate := inputDevice.DefaultSampleRate

 audioStream, err := portaudio.OpenDefaultStream(inputChannels, outputChannels, sampleRate, len(framesPerBuffer), framesPerBuffer)
 errCheck(err)

 // setup Wave file writer

 param := wave.WriterParam{
 Out: waveFile,
 Channel: inputChannels,
 SampleRate: int(sampleRate),
 BitsPerSample: 8, // if 16, change to WriteSample16()
 }

 waveWriter, err = wave.NewWriter(param)
 errCheck(err)

 // start reading from microphone
 errCheck(audioStream.Start())
 for {
 if !stopAudio { // to prevent segmentation fault
 audioStream.Read()

 fmt.Printf("\rRecording video and audio now. Wave or say something to your microphone! [%v]", ticker[rand.Intn(len(ticker)-1)])

 // write to wave file
 _, err := waveWriter.Write([]byte(framesPerBuffer)) // WriteSample16 for 16 bits
 errCheck(err)
 } else {
 break
 }

 }
 errCheck(audioStream.Stop())
 audioStream.Close()
 portaudio.Terminate()

 }

 func main() {

 // maximize CPU usage for multi threading
 runtime.GOMAXPROCS(runtime.NumCPU())

 // for ticker
 rand.Seed(time.Now().UnixNano())

 if len(os.Args) != 2 {
 fmt.Printf("Usage : %s <save to filename>\n", os.Args[0])
 os.Exit(0)
 }

 videoFileName := os.Args[1]

 // remove extension and add .wav
 ext := path.Ext(videoFileName)
 audioFileName := videoFileName[0:len(videoFileName)-len(ext)] + ".wav"

 fmt.Println("Press ESC key to to quit")
 // go routine to intercept ESC key
 // since opencv.WaitKey does not work on my Mac :(

 go func() {

 key := C.getch()
 fmt.Println()
 fmt.Println("Cleaning up ...")
 if key == 27 {

 // stop audio first
 stopAudio = true
 waveWriter.Close()
 fmt.Println("Audio closed.")

 // now we merge the video and audio files

 fmt.Println("Number of frames : ", frameCounter)

 // remember.... we are calculating Frames Per [Second]
 elapsedTime := time.Since(startTime).Seconds()
 fmt.Println("Elapsed time in seconds : ", elapsedTime)
 recordedFPS := float64(frameCounter) / elapsedTime
 fmt.Println("Recorded FPS : ", recordedFPS)

 // stop video
 stopVideo = true
 win.Destroy()
 fmt.Println("Video closed.")

 // now we merge the video and audio files

 // taken from https://github.com/JRodrigoF/AVrecordeR/blob/master/AVrecordeR.py
 // If the fps rate was higher/lower than expected, re-encode it to the expected

 if math.Abs(recordedFPS-6) >= 0.01 {

 fmt.Println("Re-encoding")
 recordedFPSstring := fmt.Sprintf("%0.2f", recordedFPS)
 fmt.Println("recordedFPS string : ", recordedFPSstring)

 // use -loglevel quiet to suppress ffmpeg errors and exit status 1
 // which will kill our program

 os.Remove("temp_" + videoFileName)

 // having white spaces in exec.Command() 
 // will return non-zero exit status. Golang does not have shell=true yet
 //cmd := exec.Command("ffmpeg", "-r "+recordedFPSstring+" -i "+videoFileName+" -r 6 temp_"+videoFileName)

 cmd := exec.Command("ffmpeg", "-r", recordedFPSstring, "-i", videoFileName, "-r", "6", "temp_"+videoFileName)
 err := cmd.Run()
 fmt.Println("Error from ffmpeg re-encoding : ", err)

 fmt.Println("Muxing")
 //cmd = exec.Command("ffmpeg", "-i temp_"+videoFileName+" -i "+audioFileName+" final_"+videoFileName)
 cmd = exec.Command("ffmpeg", "-i", "temp_"+videoFileName, "-i", audioFileName, "final_"+videoFileName)
 ffmpegdata, err := cmd.Output()
 fmt.Println("Error from ffmpeg muxing : ", err)
 fmt.Println(ffmpegdata)

 } else {
 fmt.Println("Normal recording and muxing")

 cmd := exec.Command("ffmpeg", "-i", videoFileName+".tmp", "-i", audioFileName, "final_"+videoFileName)
 fmt.Println(cmd)
 cmd.Run()
 //errCheck(err)
 }

 fmt.Println("Play final_", videoFileName, "with a video player to see the result!")
 os.Exit(0)
 }
 }()

 // almost... almost simultaneously...
 go audioFeed(audioFileName)
 videoFeed(videoFileName)

 // loop forever until escape key is pressed
 select {}
 }

---

NOTES

I reckon there are few better solutions out there.

One is at https://ffmpeg.org/ffmpeg.html#toc-Video-and-Audio-grabbing

and another is a pure Golang solution at https://github.com/3d0c/gmf/blob/master/examples/encoding_mp4.go. However, my attempts to run it generated these error messages:

[mp4 @ 0x600da00] Using AVStream.codec.timebase as a timebase hint to the muxer is deprecated. Set AVStream.timebase instead. [mp4 @ 0x600da00] Using AVStream.codec to pass codec parameters to muxers is deprecated, use AVStream.codecpar instead. [mp4 @ 0x600da00] Using AVStream.codec.timebase as a timebase hint to the muxer is deprecated. Set AVStream.timebase instead. [mp4 @ 0x600da00] Using AVStream.codec to pass codec parameters to muxers is deprecated, use AVStream.codecpar instead.

Happy coding!

References:

https://socketloop.com/tutorials/golang-record-voice-audio-from-microphone-to-wav-file

https://github.com/JRodrigoF/AVrecordeR/blob/master/AVrecordeR.py

https://ffmpeg.org/ffmpeg.html

https://socketloop.com/tutorials/golang-save-webcamera-frames-to-video-file