Golang : Upload big file (larger than 100MB) to AWS S3 with multipart upload

At my previous tutorial on uploading file to Amazon Web Services S3, a reader asked if the method in the tutorial is recommended for uploading very large files.

After some digging at the AWS official S3 documentation... the answer is no. AWS recommendation is to use Multipart method to upload any file larger than 100MB.

For this tutorial, we will first explore how to use multi.PutAll and then followed by multi.PutPart. The PutAll method is simple and straightforward to use if you don't plan to know how to chop up a file to 5MB per piece and upload piece by piece. For the PutPart method, it is more complicated but I guess there will be some situation which this method is preferred over PutAll. (See the advantages listed in http://docs.aws.amazon.com/AmazonS3/latest/dev/uploadobjusingmpu.html )

Before executing the code below.....please :

go get launchpad.net/goamz/s3

go get launchpad.net/goamz/aws

first.

Here we go, the PutAll method :

 package main

 import (
 "bufio"
 "fmt"
 "launchpad.net/goamz/aws"
 "launchpad.net/goamz/s3"
 "net/http"
 "os"
 )

 func main() {
 AWSAuth := aws.Auth{
 AccessKey: "", // change this to yours
 SecretKey: "",
 }

 region := aws.USEast
 // change this to your AWS region
 // click on the bucketname in AWS control panel and click Properties
 // the region for your bucket should be under "Static Website Hosting" tab

 connection := s3.New(AWSAuth, region)

 bucket := connection.Bucket("") // change this your bucket name

 s3path := "example/somebigfile" // this is the target file and location in S3

 fileToBeUploaded := "somebigfile" // AWS recommends multipart upload for file bigger than 100MB

 file, err := os.Open(fileToBeUploaded)

 if err != nil {
 fmt.Println(err)
 os.Exit(1)
 }

 defer file.Close()

 fileInfo, _ := file.Stat()
 var fileSize int64 = fileInfo.Size()
 bytes := make([]byte, fileSize)

 // read into buffer
 buffer := bufio.NewReader(file)
 _, err = buffer.Read(bytes)

 // then we need to determine the file type
 // see https://www.socketloop.com/tutorials/golang-how-to-verify-uploaded-file-is-image-or-allowed-file-types

 filetype := http.DetectContentType(bytes)

 // set up for multipart upload
 multi, err := bucket.InitMulti(s3path, filetype, s3.ACL("public-read"))

 if err != nil {
 fmt.Println(err)
 os.Exit(1)
 }

 // this is for PutPart ( see https://godoc.org/launchpad.net/goamz/s3#Multi.PutPart)

 // calculate the number of parts by dividing up the file size by 5MB
 const fileChunk = 5242880 // 5MB in bytes

 parts, err := multi.PutAll(file, fileChunk)

 if err != nil {
 fmt.Println(err)
 os.Exit(1)
 }

 err = multi.Complete(parts)

 if err != nil {
 fmt.Println(err)
 os.Exit(1)
 }

 fmt.Println("PutAll upload completes")

 }

and the PutPart method :

 package main

 import (
 "bufio"
 "fmt"
 "launchpad.net/goamz/aws"
 "launchpad.net/goamz/s3"
 "math"
 "net/http"
 "os"
 )

 func main() {
 AWSAuth := aws.Auth{
 AccessKey: "", // change this to yours
 SecretKey: "",
 }

 region := aws.USEast
 // change this to your AWS region
 // click on the bucketname in AWS control panel and click Properties
 // the region for your bucket should be under "Static Website Hosting" tab

 connection := s3.New(AWSAuth, region)

 bucket := connection.Bucket("") // change this your bucket name

 s3path := "example/somebigfile" // this is the target file and location in S3

 fileToBeUploaded := "somebigfile" // AWS recommends multipart upload for file bigger than 100MB

 // NOTE : If the filesize is smaller than 5MB ( as defined in fileChunk below)
 // you will get this error message :
 // "The XML you provided was not well-formed or did not validate against our published schema"

 file, err := os.Open(fileToBeUploaded)

 if err != nil {
 fmt.Println(err)
 os.Exit(1)
 }

 defer file.Close()

 fileInfo, _ := file.Stat()
 var fileSize int64 = fileInfo.Size()
 bytes := make([]byte, fileSize)

 // read into buffer
 buffer := bufio.NewReader(file)
 _, err = buffer.Read(bytes)

 // then we need to determine the file type
 // see https://www.socketloop.com/tutorials/golang-how-to-verify-uploaded-file-is-image-or-allowed-file-types

 filetype := http.DetectContentType(bytes)

 // set up for multipart upload
 multi, err := bucket.InitMulti(s3path, filetype, s3.ACL("public-read"))

 if err != nil {
 fmt.Println(err)
 os.Exit(1)
 }

 // this is for PutPart ( see https://godoc.org/launchpad.net/goamz/s3#Multi.PutPart)

 // calculate the number of parts by dividing up the file size by 5MB
 const fileChunk = 5242880 // 5MB in bytes

 // how many parts to process ??
 totalPartsNum := uint64(math.Ceil(float64(fileSize) / float64(fileChunk)))

 parts := []s3.Part{} // collect all the parts for upload completion 

 fmt.Println("Uploading...")

 for i := uint64(0); i < totalPartsNum; i++ {

 partSize := int(math.Min(fileChunk, float64(fileSize-int64(i*fileChunk))))
 partBuffer := make([]byte, partSize)

 file.Read(partBuffer)
 part, err := multi.PutPart(int(i), file) // write to S3 bucket part by part
 fmt.Printf("Processing %d part of %d and uploaded %d bytes.\n ", int(i), int(totalPartsNum), int(part.Size))

 parts = append(parts, part)

 if err != nil {
 fmt.Printf("Uploading parts of file error :i %s\n ", err)
 os.Exit(1)
 }

 }

 err = multi.Complete(parts)

 if err != nil {
 fmt.Println("Complete parts error %s\n", err)
 os.Exit(1)
 }

 fmt.Println("\n\nPutPart upload completed")

 }

sample output :

Uploading...

Processing 1 part of 5 and uploaded 25753984 bytes.

Processing 2 part of 5 and uploaded 25753984 bytes.

Processing 3 part of 5 and uploaded 25753984 bytes.

Processing 4 part of 5 and uploaded 25753984 bytes.

PutPart upload completed

References :

http://docs.aws.amazon.com/AmazonS3/latest/dev/uploadobjusingmpu.html

https://godoc.org/launchpad.net/goamz/s3#Multi.PutPart

https://godoc.org/launchpad.net/goamz/s3#Multi.PutAll