file

package
v0.0.0-...-6067653 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Aug 11, 2025 License: Apache-2.0 Imports: 20 Imported by: 0

README

file

Secure file upload and storage abstraction with support for local filesystem and S3-compatible backends.

Features

  • Unified Storage interface for local filesystem and S3
  • Built-in security features (path traversal protection, MIME validation)
  • File type detection (images, videos, audio, PDFs)
  • Content-based validation to prevent spoofing attacks

Installation

import "github.com/dmitrymomot/saaskit/pkg/file"

Usage

package main

import (
    "context"
    "net/http"
    
    "github.com/dmitrymomot/saaskit/pkg/file"
)

func handleUpload(w http.ResponseWriter, r *http.Request) {
    // Parse multipart form (32MB limit)
    r.ParseMultipartForm(32 << 20)
    fh, _, err := r.FormFile("avatar")
    if err != nil {
        http.Error(w, err.Error(), http.StatusBadRequest)
        return
    }
    defer fh.Close()
    
    // Validate file
    if err := file.ValidateSize(fh, 5<<20); err != nil { // 5MB limit
        http.Error(w, err.Error(), http.StatusBadRequest)
        return
    }
    
    if !file.IsImage(fh) {
        http.Error(w, "only images allowed", http.StatusBadRequest)
        return
    }
    
    // Create storage
    storage, err := file.NewLocalStorage("./uploads", "/files/")
    if err != nil {
        http.Error(w, err.Error(), http.StatusInternalServerError)
        return
    }
    
    // Save file
    ctx := context.Background()
    fileInfo, err := storage.Save(ctx, fh, "avatars/user123.jpg")
    if err != nil {
        http.Error(w, err.Error(), http.StatusInternalServerError)
        return
    }
    
    // Return public URL
    w.Write([]byte(storage.URL(fileInfo.RelativePath)))
}

Common Operations

Local Storage
// Create local storage with options
storage, err := file.NewLocalStorage(
    "./uploads",     // Base directory
    "/files/",       // URL prefix
    file.WithLocalUploadTimeout(30*time.Second),
)

// List files
entries, err := storage.List(ctx, "avatars/")

// Check existence
exists := storage.Exists(ctx, "avatars/user123.jpg")

// Delete file
err = storage.Delete(ctx, "avatars/user123.jpg")

// Delete directory
err = storage.DeleteDir(ctx, "avatars/")
S3 Storage
// Create S3 storage
storage, err := file.NewS3Storage(ctx, file.S3Config{
    Bucket:      "my-bucket",
    Region:      "us-east-1", 
    AccessKeyID: os.Getenv("AWS_ACCESS_KEY_ID"),
    SecretKey:   os.Getenv("AWS_SECRET_ACCESS_KEY"),
    BaseURL:     "https://cdn.example.com", // Optional CDN URL
})

// Same interface as local storage
fileInfo, err := storage.Save(ctx, fh, "uploads/doc.pdf")
url := storage.URL(fileInfo.RelativePath)
File Validation
// Validate MIME types
err := file.ValidateMIMEType(fh, "image/jpeg", "image/png", "application/pdf")

// Type checking helpers
if file.IsImage(fh) { /* ... */ }
if file.IsVideo(fh) { /* ... */ }
if file.IsAudio(fh) { /* ... */ }
if file.IsPDF(fh) { /* ... */ }

// Get file hash
hash, err := file.Hash(fh, nil) // Uses SHA256 by default

// Sanitize filename (prevents path traversal)
safe := file.SanitizeFilename("../../../etc/passwd") // Returns "passwd"

Error Handling

// Package errors:
var (
    ErrNilFileHeader      = errors.New("file header is nil")
    ErrInvalidPath        = errors.New("invalid path")
    ErrFileNotFound       = errors.New("file not found")
    ErrDirectoryNotFound  = errors.New("directory not found")
    ErrFileTooLarge       = errors.New("file size exceeds maximum allowed size")
    ErrMIMETypeNotAllowed = errors.New("MIME type is not allowed")
)

// Usage:
if errors.Is(err, file.ErrFileNotFound) {
    // Handle missing file
}

if errors.Is(err, file.ErrFileTooLarge) {
    // Handle oversized file
}

API Documentation

# Full API documentation
go doc github.com/dmitrymomot/saaskit/pkg/file

# Specific function or type
go doc github.com/dmitrymomot/saaskit/pkg/file.Storage

Notes

  • Path traversal attacks are prevented automatically in both storage backends
  • MIME type detection reads file content, not just extensions (prevents spoofing)
  • S3Storage supports any S3-compatible service (MinIO, DigitalOcean Spaces, etc.)
  • Large file uploads should use context with timeout to prevent resource exhaustion

Documentation

Overview

Package file provides a comprehensive file management system with support for local and S3 storage backends.

The package offers a unified Storage interface for file operations across different backends, along with utilities for file validation, content detection, and security checks. It's designed to handle common file upload scenarios in web applications with built-in protection against common security vulnerabilities like path traversal and MIME type spoofing.

Architecture

The package is built around the Storage interface which provides a consistent API for:

  • Saving files with automatic path sanitization
  • Deleting files and directories
  • Checking file existence
  • Listing directory contents
  • Generating public URLs

Two implementations are provided:

  • LocalStorage: For filesystem-based storage
  • S3Storage: For AWS S3 and S3-compatible services (MinIO, Wasabi, etc.)

Usage

Basic file upload handling with validation:

import "github.com/dmitrymomot/saaskit/pkg/file"

// Create storage backend
storage := file.NewLocalStorage("/uploads", "https://example.com/files")

// In HTTP handler
fh := r.MultipartForm.File["avatar"][0]

// Validate file
if err := file.ValidateSize(fh, 5<<20); err != nil { // 5MB limit
	return err
}

if err := file.ValidateMIMEType(fh, "image/jpeg", "image/png"); err != nil {
	return err
}

// Save file
fileInfo, err := storage.Save(ctx, fh, "avatars/user123.jpg")
if err != nil {
	return err
}

// Get public URL
url := storage.URL(fileInfo.RelativePath)

Using S3 storage:

storage, err := file.NewS3Storage(ctx, file.S3Config{
	Bucket:      "my-bucket",
	Region:      "us-east-1",
	AccessKeyID: "key",
	SecretKey:   "secret",
})
if err != nil {
	return err
}

// Same Storage interface methods work with S3
fileInfo, err := storage.Save(ctx, fh, "uploads/document.pdf")

File Validation

The package provides several validation utilities:

// Check file type
if file.IsImage(fh) {
	// Process image
}

// Validate size (prevents DoS from large uploads)
err := file.ValidateSize(fh, 10<<20) // 10MB max

// Validate MIME type (uses content detection, not extension)
err := file.ValidateMIMEType(fh, "application/pdf", "application/msword")

// Get file hash for deduplication
hash, err := file.Hash(fh, sha256.New())

Security Considerations

The package implements several security measures:

  • Path sanitization prevents directory traversal attacks
  • MIME type detection uses file content, not extension
  • Automatic filename sanitization removes dangerous characters
  • Size validation prevents resource exhaustion
  • Support for separate storage and public URL paths

Configuration

LocalStorage configuration:

  • BaseDir: Root directory for file storage
  • BaseURL: Public URL prefix for generating file URLs
  • DirPerm: Directory creation permissions (default: 0755)

S3Storage configuration:

  • Standard AWS credentials (key, secret, region)
  • Custom endpoints for S3-compatible services
  • Path-style URLs for MinIO compatibility
  • Custom CDN base URLs
  • Upload timeouts for large files

Error Handling

The package defines specific errors for different failure scenarios:

fileInfo, err := storage.Save(ctx, fh, "test.jpg")
if errors.Is(err, file.ErrFileTooLarge) {
	// File exceeds size limit
} else if errors.Is(err, file.ErrMIMETypeNotAllowed) {
	// Invalid file type
} else if errors.Is(err, file.ErrInvalidPath) {
	// Path contains dangerous characters
}

S3-specific errors are mapped to generic file errors for consistency:

  • NoSuchBucket -> ErrDirectoryNotFound
  • NoSuchKey -> ErrFileNotFound
  • AccessDenied -> ErrAccessDenied

Performance Considerations

- File content is streamed during uploads to minimize memory usage - S3Storage supports configurable timeouts for large file uploads - Directory listings use pagination to handle large directories - MIME type detection reads only first 512 bytes - Hash calculation streams file content without loading into memory

Examples

See the package examples and README.md for detailed usage patterns including S3-compatible services, CDN integration, and batch operations.

Index

Constants

This section is empty.

Variables

View Source
var (
	// Security and validation errors
	ErrNilFileHeader = errors.New("file header is nil")
	ErrInvalidPath   = errors.New("invalid path") // Prevents path traversal attacks

	// File system errors
	ErrFileNotFound      = errors.New("file not found")
	ErrDirectoryNotFound = errors.New("directory not found")
	ErrNotDirectory      = errors.New("path is not a directory")
	ErrIsDirectory       = errors.New("path is a directory")

	// File validation errors
	ErrFileTooLarge       = errors.New("file size exceeds maximum allowed size")
	ErrMIMETypeNotAllowed = errors.New("MIME type is not allowed")

	// I/O operation errors - wrapped with context for debugging
	ErrFailedToOpenFile        = errors.New("failed to open file")
	ErrFailedToReadFile        = errors.New("failed to read file")
	ErrFailedToWriteFile       = errors.New("failed to write file")
	ErrFailedToCreateFile      = errors.New("failed to create file")
	ErrFailedToDeleteFile      = errors.New("failed to delete file")
	ErrFailedToCreateDirectory = errors.New("failed to create directory")
	ErrFailedToDeleteDirectory = errors.New("failed to delete directory")
	ErrFailedToReadDirectory   = errors.New("failed to read directory")
	ErrFailedToStatPath        = errors.New("failed to stat path")
	ErrFailedToGetAbsolutePath = errors.New("failed to get absolute path")
	ErrFailedToDetectMIMEType  = errors.New("failed to detect MIME type")
	ErrFailedToHashFile        = errors.New("failed to hash file")

	// S3-specific errors for proper error classification
	ErrBucketNotFound     = errors.New("bucket not found")
	ErrAccessDenied       = errors.New("access denied")
	ErrRequestTimeout     = errors.New("request timed out")
	ErrServiceUnavailable = errors.New("service temporarily unavailable") // Used for throttling and retries
	ErrInvalidObjectState = errors.New("invalid object state")

	// Context and cancellation errors
	ErrOperationTimeout  = errors.New("operation timed out")
	ErrOperationCanceled = errors.New("operation canceled")

	// Configuration errors
	ErrPaginatorNil       = errors.New("paginator factory returned nil") // Testing support
	ErrInvalidConfig      = errors.New("invalid configuration")
	ErrFailedToLoadConfig = errors.New("failed to load AWS config")
)

Functions

func GetExtension

func GetExtension(fh *multipart.FileHeader) string

GetExtension returns the file extension including the dot.

Example:

ext := file.GetExtension(fh) // ".jpg"

func GetMIMEType

func GetMIMEType(fh *multipart.FileHeader) (string, error)

GetMIMEType detects the MIME type by reading the file content. Uses http.DetectContentType which reads the first 512 bytes to identify file types based on magic bytes rather than trusting file extensions (prevents spoofing). Resets file position to allow subsequent reads of the same file.

func Hash

func Hash(fh *multipart.FileHeader, h hash.Hash) (string, error)

Hash calculates the hash of the file content. Defaults to SHA256 for security and compatibility with most systems. Used for deduplication, integrity verification, and content-based addressing.

Example:

hashStr, err := file.Hash(fh, sha256.New())

func IsAudio

func IsAudio(fh *multipart.FileHeader) bool

IsAudio checks if the file is an audio file based on MIME type. Uses the same dual validation approach as IsImage for security.

func IsImage

func IsImage(fh *multipart.FileHeader) bool

IsImage checks if the file is an image based on MIME type. Falls back to extension check if MIME type detection fails to handle cases where http.DetectContentType can't read the file or returns generic types. This dual validation prevents bypass attacks using renamed extensions.

Example:

if file.IsImage(fh) {
    // Process image file
}

func IsPDF

func IsPDF(fh *multipart.FileHeader) bool

IsPDF checks if the file is a PDF.

func IsVideo

func IsVideo(fh *multipart.FileHeader) bool

IsVideo checks if the file is a video based on MIME type. Uses the same dual validation approach as IsImage for security.

func ReadAll

func ReadAll(fh *multipart.FileHeader) ([]byte, error)

ReadAll reads the entire file content into memory. Use with caution for large files as it can cause memory exhaustion. Consider streaming approaches for files larger than available memory.

func SanitizeFilename

func SanitizeFilename(filename string) string

SanitizeFilename removes any path components and dangerous characters from a filename to prevent path traversal attacks and other security issues. Returns "unnamed" for empty or special directory references.

Example:

safe := file.SanitizeFilename("../../../etc/passwd") // Returns "passwd"
safe = file.SanitizeFilename("C:\\Windows\\file.txt") // Returns "file.txt"

func ValidateMIMEType

func ValidateMIMEType(fh *multipart.FileHeader, allowedTypes ...string) error

ValidateMIMEType checks if the file's MIME type is in the allowed list. Uses actual content detection to prevent MIME type spoofing attacks. Pass no types to allow all MIME types (useful for generic file storage).

Example:

if err := file.ValidateMIMEType(fh, "image/jpeg", "image/png"); err != nil {
    return err
}

func ValidateSize

func ValidateSize(fh *multipart.FileHeader, maxBytes int64) error

ValidateSize checks if the file size is within the allowed limit. Note: For streamed uploads, FileHeader.Size may be 0. Storage implementations should perform actual size validation during upload to prevent DoS attacks from oversized files that bypass this check.

Example:

if err := file.ValidateSize(fh, 5<<20); err != nil { // 5MB limit
    return err
}

Types

type Entry

type Entry struct {
	Name  string
	Path  string
	IsDir bool
	Size  int64
}

Entry represents a file or directory entry.

type File

type File struct {
	Filename     string
	Size         int64
	MIMEType     string
	Extension    string
	AbsolutePath string
	RelativePath string
}

File represents stored file metadata.

type LocalOption

type LocalOption func(*LocalStorage)

LocalOption defines a function that configures LocalStorage.

func WithLocalUploadTimeout

func WithLocalUploadTimeout(timeout time.Duration) LocalOption

WithLocalUploadTimeout sets the timeout for upload operations. Prevents hanging uploads from consuming resources indefinitely. If not set, relies on context deadline from caller.

type LocalStorage

type LocalStorage struct {
	// contains filtered or unexported fields
}

LocalStorage implements Storage interface for local filesystem. All operations are confined to baseDir to prevent path traversal attacks. Safe for concurrent use with proper file locking by the OS.

func NewLocalStorage

func NewLocalStorage(baseDir, baseURL string, opts ...LocalOption) (*LocalStorage, error)

NewLocalStorage creates a new local filesystem storage. baseDir is resolved to absolute path and created if it doesn't exist. baseURL is used for generating public URLs (e.g., "/files/"). All file operations are confined to baseDir to prevent path traversal attacks.

func (*LocalStorage) Delete

func (s *LocalStorage) Delete(ctx context.Context, path string) error

Delete removes a single file. Verifies the target is a file, not a directory, to prevent accidental data loss.

func (*LocalStorage) DeleteDir

func (s *LocalStorage) DeleteDir(ctx context.Context, path string) error

DeleteDir recursively removes a directory and all its contents. Verifies the target is a directory to prevent accidental file deletion.

func (*LocalStorage) Exists

func (s *LocalStorage) Exists(ctx context.Context, path string) bool

Exists checks if a file or directory exists. Returns false for invalid paths or on context cancellation.

func (*LocalStorage) List

func (s *LocalStorage) List(ctx context.Context, dir string) ([]Entry, error)

List returns all entries in a directory (non-recursive). Checks context cancellation periodically during iteration to handle large directories.

func (*LocalStorage) Save

func (s *LocalStorage) Save(ctx context.Context, fh *multipart.FileHeader, path string) (*File, error)

Save stores a file to the local filesystem. Uses buffered I/O with context cancellation support to handle large files efficiently while allowing early termination. Cleans up partial files on errors.

func (*LocalStorage) URL

func (s *LocalStorage) URL(path string) string

URL returns the public URL for a file.

type S3Client

type S3Client interface {
	PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error)
	HeadObject(ctx context.Context, params *s3.HeadObjectInput, optFns ...func(*s3.Options)) (*s3.HeadObjectOutput, error)
	ListObjectsV2(ctx context.Context, params *s3.ListObjectsV2Input, optFns ...func(*s3.Options)) (*s3.ListObjectsV2Output, error)
	DeleteObject(ctx context.Context, params *s3.DeleteObjectInput, optFns ...func(*s3.Options)) (*s3.DeleteObjectOutput, error)
	DeleteObjects(ctx context.Context, params *s3.DeleteObjectsInput, optFns ...func(*s3.Options)) (*s3.DeleteObjectsOutput, error)
}

S3Client defines the interface for S3 operations used by S3Storage.

type S3Config

type S3Config struct {
	Bucket         string
	Region         string
	AccessKeyID    string
	SecretKey      string
	Endpoint       string // For S3-compatible services like MinIO, Wasabi
	BaseURL        string // Custom CDN or public URL base (auto-generated if empty)
	ForcePathStyle bool   // Required for MinIO and some S3-compatible services
}

S3Config contains configuration for S3 storage.

type S3ListObjectsV2Paginator

type S3ListObjectsV2Paginator interface {
	HasMorePages() bool
	NextPage(ctx context.Context, optFns ...func(*s3.Options)) (*s3.ListObjectsV2Output, error)
}

S3ListObjectsV2Paginator defines the interface for paginated list operations.

type S3Option

type S3Option func(*s3Options)

S3Option defines a function that configures S3Storage.

func WithHTTPClient

func WithHTTPClient(client *http.Client) S3Option

WithHTTPClient sets a custom HTTP client for S3 requests. Useful for custom timeout, proxy, or TLS configuration.

func WithPaginatorFactory

func WithPaginatorFactory(factory func(client S3Client, params *s3.ListObjectsV2Input) S3ListObjectsV2Paginator) S3Option

WithPaginatorFactory sets a custom paginator factory. Essential for testing pagination behavior with mock clients.

func WithS3Client

func WithS3Client(client S3Client) S3Option

WithS3Client sets a custom pre-configured S3 client. Primarily used for testing with mocks, but also allows advanced client customization.

func WithS3ClientOption

func WithS3ClientOption(option func(*s3.Options)) S3Option

WithS3ClientOption adds a custom S3 client option.

func WithS3ConfigOption

func WithS3ConfigOption(option func(*config.LoadOptions) error) S3Option

WithS3ConfigOption adds a custom AWS config option.

func WithS3UploadTimeout

func WithS3UploadTimeout(timeout time.Duration) S3Option

WithS3UploadTimeout sets the timeout for upload operations. Prevents hanging uploads from consuming resources indefinitely. If not set, relies on context deadline from caller.

type S3Storage

type S3Storage struct {
	// contains filtered or unexported fields
}

S3Storage implements Storage interface for Amazon S3 and S3-compatible services. Thread-safe with automatic retry and error classification for reliable operation.

func NewS3Storage

func NewS3Storage(ctx context.Context, cfg S3Config, opts ...S3Option) (*S3Storage, error)

NewS3Storage creates a new S3 storage instance. Auto-generates baseURL if not provided, supports both AWS S3 and S3-compatible services.

func (*S3Storage) Delete

func (s *S3Storage) Delete(ctx context.Context, path string) error

Delete removes a single file from S3. Verifies existence before deletion to provide consistent error handling.

func (*S3Storage) DeleteDir

func (s *S3Storage) DeleteDir(ctx context.Context, dir string) error

DeleteDir removes all objects with the given prefix from S3. Uses batch deletion (1000 objects per request) for efficiency on large directories.

func (*S3Storage) Exists

func (s *S3Storage) Exists(ctx context.Context, path string) bool

Exists checks if an object exists in S3.

func (*S3Storage) List

func (s *S3Storage) List(ctx context.Context, dir string) ([]Entry, error)

List returns all entries in a directory (non-recursive). Uses S3 delimiter to simulate directory structure and avoid deep recursion.

func (*S3Storage) Save

func (s *S3Storage) Save(ctx context.Context, fh *multipart.FileHeader, path string) (*File, error)

Save stores a file to S3. Validates path to prevent S3 key injection attacks and sets proper Content-Type.

func (*S3Storage) URL

func (s *S3Storage) URL(path string) string

URL returns the public URL for a file.

type Storage

type Storage interface {
	// Save stores a file and returns metadata.
	Save(ctx context.Context, fh *multipart.FileHeader, path string) (*File, error)
	// Delete removes a single file.
	Delete(ctx context.Context, path string) error
	// DeleteDir recursively removes a directory and all its contents.
	DeleteDir(ctx context.Context, path string) error
	// Exists checks if a file or directory exists.
	Exists(ctx context.Context, path string) bool
	// List returns all entries in a directory (non-recursive).
	List(ctx context.Context, dir string) ([]Entry, error)
	// URL returns the public URL for a file.
	URL(path string) string
}

Storage interface for different backends.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL