Streaming File Copy with Progress and SHA-256 Verification
Copies large files with constant memory usage while reporting progress and verifying integrity with SHA-256.
main.go
1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980
package main
import (
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"log"
"os"
"time"
)
func copyWithProgress(ctx context.Context, srcPath, dstPath string) (int64, string, error) {
src, err := os.Open(srcPath)
if err != nil {
return 0, "", fmt.Errorf("open src: %w", err)
}
defer src.Close()
dst, err := os.Create(dstPath)
if err != nil {
return 0, "", fmt.Errorf("create dst: %w", err)
}
defer func() {
dst.Sync()
dst.Close()
}()
hasher := sha256.New()
buf := make([]byte, 256*1024)
var total int64
ticker := time.NewTicker(500 * time.Millisecond)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return total, "", ctx.Err()
case <-ticker.C:
log.Printf("copied %d bytes", total)
default:
}
n, rerr := src.Read(buf)
if n > 0 {
chunk := buf[:n]
hasher.Write(chunk)
if _, werr := dst.Write(chunk); werr != nil {
return total, "", fmt.Errorf("write dst: %w", werr)
}
total += int64(n)
}
if rerr != nil {
if rerr == io.EOF {
break
}
return total, "", fmt.Errorf("read src: %w", rerr)
}
}
if err := dst.Sync(); err != nil {
return total, "", fmt.Errorf("sync dst: %w", err)
}
sum := hex.EncodeToString(hasher.Sum(nil))
return total, sum, nil
}
func main() {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
bytes, sha, err := copyWithProgress(ctx, "input.bin", "output.bin")
if err != nil {
log.Fatalf("copy failed: %v", err)
}
log.Printf("done bytes=%d sha256=%s", bytes, sha)
}How It Works
Copies large files with constant memory usage while reporting progress and verifying integrity with SHA-256.
Opens source and destination files, streams bytes through a buffer, updates a hash concurrently, prints percentage progress based on file size, and supports cancellation via context; on completion compares the hash to ensure it matches.
Key Concepts
- 1Streamed copy avoids loading the entire file into memory.
- 2Inline hashing provides integrity verification.
- 3Context cancellation lets callers abort long transfers.
When to Use This Pattern
- Transferring large assets or backups safely.
- Deploy pipelines that need progress feedback.
- Mirroring files across disks or hosts with verification.
Best Practices
- Check file sizes and permissions before starting to copy.
- Choose buffer sizes appropriate to storage throughput.
- Handle partial copies on errors by cleaning up or resuming.
Go Version1.18+
Difficultyintermediate
Production ReadyYes
Lines of Code80