Optimizing Large File Transfers in Linux with Go TCP/Syscall

In this article, we’ll explore some approaches and tips for sending large files over TCP in linux using Go, taking into account the constraints of small devices and the importance of efficient and reliable file transmission.

Naive approach

func sendFile(file *os.File, conn net.Conn) error {
    // Get file stat
    fileInfo, _ := file.Stat()

    // Send the file size
    sizeBuf := make([]byte, 8)
    binary.LittleEndian.PutUint64(sizeBuf, uint64(fileInfo.Size()))
    _, err := conn.Write(sizeBuf)
    if err != nil {
        return err
    }

    // Send the file contents by chunks
    buf := make([]byte, 1024)
    for {
        n, err := file.Read(buf)
        if err == io.EOF {
            break
        }

        _, err = conn.Write(buf[:n])
        if err != nil {
            fmt.Println("error writing to the conn:", err)
            break
        }
    }

    return nil
}

Using a specialized syscall ‘sendfile’

func sendFile(file *os.File, conn net.Conn) error {
    // Get file stat
    fileInfo, _ := file.Stat()

    // Send the file size
    sizeBuf := make([]byte, 8)
    binary.LittleEndian.PutUint64(sizeBuf, uint64(fileInfo.Size()))
    if _, err := conn.Write(sizeBuf); err != nil {
        return err
    }

    tcpConn, ok := conn.(*net.TCPConn)
    if !ok {
        return errors.New("TCPConn error")
    }

    tcpF, err := tcpConn.File()
    if err != nil {
        return err
    }

    // Send the file contents
    _, err = syscall.Sendfile(int(tcpF.Fd()), int(file.Fd()), nil, int(fileInfo.Size()))
    return err
}

The sendfile syscall is more efficient in transferring data than standard read and write methods. By bypassing the app buffer, the data moves directly from the read buffer to the socket buffer, reducing the number of data copies and context switches and improving performance. Furthermore, the process could requires less CPU intervention, allowing quicker data transfer and freeing up CPU for other tasks.

The sendfile syscall is known as a "zero-copy" method because it transfers data from one file descriptor to another without the need for an intermediate data copy in user-space memory.

func sendFile(file *os.File, conn net.Conn) error {
    // Get file stat
    fileInfo, _ := file.Stat()

    // Send the file size
    sizeBuf := make([]byte, 8)
    binary.LittleEndian.PutUint64(sizeBuf, uint64(fileInfo.Size()))
    _, err := conn.Write(sizeBuf)
    if err != nil {
        return err
    }

    // Send the file contents
    _, err = io.Copy(conn, file)
    return err
}

The benefits of using io.Copy in Go go beyond its 32k buffer management and optimization.

func copyBuffer(dst Writer, src Reader, buf []byte) (written int64, err error) {
    ...
    if wt, ok := src.(WriterTo); ok {
        return wt.WriteTo(dst)
    }

    if rt, ok := dst.(ReaderFrom); ok {
        return rt.ReadFrom(src)
    }
    ...
}

Generate a demo file for testing

dd if=/dev/urandom of=dummy.dat bs=1M count=230 # This generate a file with size of 230MB aprox with random data!

All in One

Client

package main

import (
	"encoding/binary"
	"io"
	"log"
	"net"
	"os"
)

func receiveFile(path string, conn net.Conn) error {
	// Create the file
	file, err := os.Create(path)
	if err != nil {
		log.Fatal(err)
	}

	defer file.Close()

	// Get the file size
	sizeBuf := make([]byte, 8)
	if _, err := conn.Read(sizeBuf); err != nil {
		return err
	}

	fileSize := binary.LittleEndian.Uint64(sizeBuf)
	// Receive the file contents
	_, err = io.CopyN(file, conn, int64(fileSize))
	return err
}

func main() {
	// Connect to the server
	conn, err := net.Dial("tcp", "x.x.x.x:3000")
	if err != nil {
		log.Fatal(err)
	}
	defer conn.Close()

	// Receive the file from the server
	err = receiveFile("local.dat", conn)
	if err != nil {
		log.Fatal(err)
	}
}

sendfile-server

package main

import (
	"encoding/binary"
	"errors"
	"fmt"
	"log"
	"net"
	"os"
	"syscall"
)

func sendFile(file *os.File, conn net.Conn) error {
	// Get file stat
	fileInfo, _ := file.Stat()

	// Send the file size
	sizeBuf := make([]byte, 8)
	binary.LittleEndian.PutUint64(sizeBuf, uint64(fileInfo.Size()))
	if _, err := conn.Write(sizeBuf); err != nil {
		return err
	}

	tcpConn, ok := conn.(*net.TCPConn)
	if !ok {
		return errors.New("TCPConn error")
	}

	tcpF, err := tcpConn.File()
	if err != nil {
		return err
	}

	// Send the file contents
	_, err = syscall.Sendfile(int(tcpF.Fd()), int(file.Fd()), nil, int(fileInfo.Size()))
	return err
}

func main() {
	// Create the listener
	listener, err := net.Listen("tcp", ":3000")
	if err != nil {
		log.Fatal(err)
	}

	defer listener.Close()

	for {
		// Wait for a client to connect
		conn, err := listener.Accept()
		if err != nil {
			log.Fatal(err)
		}

		// Send the file to the client
		go func() {
			// Open the file
			file, err := os.Open("../dummy.dat")
			if err != nil {
				return
			}
			defer file.Close()

			if err := sendFile(file, conn); err != nil {
				fmt.Println(err)
			}
			conn.Close()
		}()
	}

}

naive server

func sendFile(file *os.File, conn net.Conn) error {
	// Get file stat
	fileInfo, _ := file.Stat()
	// Send the file size
	sizeBuf := make([]byte, 8)
	binary.LittleEndian.PutUint64(sizeBuf, uint64(fileInfo.Size()))
	_, err := conn.Write(sizeBuf)
	if err != nil {
		return err
	}

	// Send the file contents
	buf := make([]byte, 1024)
	for {
		n, err := file.Read(buf)
		if err == io.EOF {
			break
		}

		_, err = conn.Write(buf[:n])
		if err != nil {
			fmt.Println("error writing to the conn:", err)
			return err
		}
	}

	return nil
}

copy server

func sendFile(file *os.File, conn net.Conn) error {
	// Get file stat
	fileInfo, _ := file.Stat()

	// Send the file size
	sizeBuf := make([]byte, 8)
	binary.LittleEndian.PutUint64(sizeBuf, uint64(fileInfo.Size()))
	_, err := conn.Write(sizeBuf)
	if err != nil {
		return err
	}

	// Send the file contents
	_, err = io.Copy(conn, file)
	return err
}