feat: implement unified S3-compatible storage system

Consolidates storage backends into a single S3-compatible driver that supports: - AWS S3 (native) - Backblaze B2 (S3-compatible API) - Cloudflare R2 (S3-compatible API) - MinIO and other S3-compatible services - Local filesystem for development This replaces the previous separate B2 driver with a unified approach, reducing dependencies and complexity while adding support for more services. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
2025-07-01 23:07:44 +12:00
parent e3152d9f40
commit f99a866e13
14 changed files with 1650 additions and 2 deletions
--- a/core/api/attachment_integration_test.go
+++ b/core/api/attachment_integration_test.go
@@ -0,0 +1,306 @@
+package api
+
+import (
+	"io"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/openaccounting/oa-server/core/model"
+	"github.com/openaccounting/oa-server/core/model/types"
+	"github.com/openaccounting/oa-server/core/util"
+	"github.com/openaccounting/oa-server/core/util/id"
+	"github.com/openaccounting/oa-server/database"
+	"github.com/stretchr/testify/assert"
+	"gorm.io/driver/sqlite"
+	"gorm.io/gorm"
+)
+
+func setupTestDatabase(t *testing.T) (*gorm.DB, func()) {
+	// Create temporary database file
+	tmpDir := t.TempDir()
+	dbPath := filepath.Join(tmpDir, "test.db")
+	
+	db, err := gorm.Open(sqlite.Open(dbPath), &gorm.Config{})
+	if err != nil {
+		t.Fatalf("Failed to open database: %v", err)
+	}
+
+	// Set global DB for database package
+	database.DB = db
+
+	// Run migrations
+	err = database.AutoMigrate()
+	if err != nil {
+		t.Fatalf("Failed to run auto migrations: %v", err)
+	}
+
+	err = database.Migrate()
+	if err != nil {
+		t.Fatalf("Failed to run custom migrations: %v", err)
+	}
+
+	// Cleanup function
+	cleanup := func() {
+		sqlDB, _ := db.DB()
+		if sqlDB != nil {
+			sqlDB.Close()
+		}
+		os.RemoveAll(tmpDir)
+	}
+
+	return db, cleanup
+}
+
+func setupTestData(t *testing.T, db *gorm.DB) (orgID, userID, transactionID string) {
+	// Use hardcoded UUIDs without dashes for hex format
+	orgID = "550e8400e29b41d4a716446655440000"
+	userID = "550e8400e29b41d4a716446655440001"
+	transactionID = "550e8400e29b41d4a716446655440002"
+	accountID := "550e8400e29b41d4a716446655440003"
+
+	// Insert test data using raw SQL for reliability
+	now := time.Now()
+	
+	// Insert org
+	err := db.Exec("INSERT INTO orgs (id, inserted, updated, name, currency, `precision`, timezone) VALUES (UNHEX(?), ?, ?, ?, ?, ?, ?)",
+		orgID, now.UnixNano()/int64(time.Millisecond), now.UnixNano()/int64(time.Millisecond), "Test Org", "USD", 2, "UTC").Error
+	if err != nil {
+		t.Fatalf("Failed to insert org: %v", err)
+	}
+
+	// Insert user  
+	err = db.Exec("INSERT INTO users (id, inserted, updated, firstName, lastName, email, passwordHash, agreeToTerms, passwordReset, emailVerified, emailVerifyCode, signupSource) VALUES (UNHEX(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+		userID, now.UnixNano()/int64(time.Millisecond), now.UnixNano()/int64(time.Millisecond), "Test", "User", "test@example.com", "hashedpassword", true, "", true, "", "test").Error
+	if err != nil {
+		t.Fatalf("Failed to insert user: %v", err)
+	}
+
+	// Insert user-org relationship
+	err = db.Exec("INSERT INTO user_orgs (userId, orgId, admin) VALUES (UNHEX(?), UNHEX(?), ?)",
+		userID, orgID, false).Error
+	if err != nil {
+		t.Fatalf("Failed to insert user-org: %v", err)
+	}
+
+	// Insert account
+	err = db.Exec("INSERT INTO accounts (id, orgId, inserted, updated, name, parent, currency, `precision`, debitBalance) VALUES (UNHEX(?), UNHEX(?), ?, ?, ?, ?, ?, ?, ?)",
+		accountID, orgID, now.UnixNano()/int64(time.Millisecond), now.UnixNano()/int64(time.Millisecond), "Test Account", []byte{}, "USD", 2, true).Error
+	if err != nil {
+		t.Fatalf("Failed to insert account: %v", err)
+	}
+
+	// Insert transaction
+	err = db.Exec("INSERT INTO transactions (id, orgId, userId, inserted, updated, date, description, data, deleted) VALUES (UNHEX(?), UNHEX(?), UNHEX(?), ?, ?, ?, ?, ?, ?)",
+		transactionID, orgID, userID, now.UnixNano()/int64(time.Millisecond), now.UnixNano()/int64(time.Millisecond), now.UnixNano()/int64(time.Millisecond), "Test Transaction", "", false).Error
+	if err != nil {
+		t.Fatalf("Failed to insert transaction: %v", err)
+	}
+
+	// Insert split
+	err = db.Exec("INSERT INTO splits (transactionId, accountId, date, inserted, updated, amount, nativeAmount, deleted) VALUES (UNHEX(?), UNHEX(?), ?, ?, ?, ?, ?, ?)",
+		transactionID, accountID, now.UnixNano()/int64(time.Millisecond), now.UnixNano()/int64(time.Millisecond), now.UnixNano()/int64(time.Millisecond), 100, 100, false).Error
+	if err != nil {
+		t.Fatalf("Failed to insert split: %v", err)
+	}
+
+	return orgID, userID, transactionID
+}
+
+func createTestFile(t *testing.T) (string, []byte) {
+	content := []byte("This is a test file content for attachment testing")
+	tmpDir := t.TempDir()
+	filePath := filepath.Join(tmpDir, "test.txt")
+	
+	err := os.WriteFile(filePath, content, 0644)
+	if err != nil {
+		t.Fatalf("Failed to create test file: %v", err)
+	}
+	
+	return filePath, content
+}
+
+
+func TestAttachmentIntegration(t *testing.T) {
+	db, cleanup := setupTestDatabase(t)
+	defer cleanup()
+
+	orgID, userID, transactionID := setupTestData(t, db)
+
+	// Set up the model instance for the API handlers
+	bc := &util.StandardBcrypt{}
+	
+	// Use the existing datastore model which has the attachment implementation
+	// We need to create it with the database connection
+	datastoreModel := model.NewModel(nil, bc, types.Config{})
+	model.Instance = datastoreModel
+
+	t.Run("Database Integration Test", func(t *testing.T) {
+		// Test direct database operations first
+		filePath, originalContent := createTestFile(t)
+		defer os.Remove(filePath)
+
+		// Create attachment record directly
+		attachmentID := id.String(id.New())
+		uploadTime := time.Now()
+		
+		attachment := types.Attachment{
+			Id:            attachmentID,
+			TransactionId: transactionID,
+			OrgId:         orgID,
+			UserId:        userID,
+			FileName:      "stored_test.txt",
+			OriginalName:  "test.txt",
+			ContentType:   "text/plain",
+			FileSize:      int64(len(originalContent)),
+			FilePath:      "uploads/test/" + attachmentID + ".txt",
+			Description:   "Test attachment description",
+			Uploaded:      uploadTime,
+			Deleted:       false,
+		}
+
+		// Insert using the existing model
+		createdAttachment, err := model.Instance.CreateAttachment(&attachment)
+		assert.NoError(t, err)
+		assert.NotNil(t, createdAttachment)
+		assert.Equal(t, attachmentID, createdAttachment.Id)
+
+		// Verify database persistence
+		var dbAttachment types.Attachment
+		err = db.Raw("SELECT HEX(id) as id, HEX(transactionId) as transactionId, HEX(orgId) as orgId, HEX(userId) as userId, fileName, originalName, contentType, fileSize, filePath, description, uploaded, deleted FROM attachment WHERE HEX(id) = ?", attachmentID).Scan(&dbAttachment).Error
+		assert.NoError(t, err)
+		
+		assert.Equal(t, attachmentID, dbAttachment.Id)
+		assert.Equal(t, transactionID, dbAttachment.TransactionId)
+		assert.Equal(t, "Test attachment description", dbAttachment.Description)
+
+		// Test retrieval
+		retrievedAttachment, err := model.Instance.GetAttachment(attachmentID, transactionID, orgID, userID)
+		assert.NoError(t, err)
+		assert.NotNil(t, retrievedAttachment)
+		assert.Equal(t, attachmentID, retrievedAttachment.Id)
+
+		// Test listing by transaction
+		attachments, err := model.Instance.GetAttachmentsByTransaction(transactionID, orgID, userID)
+		assert.NoError(t, err)
+		assert.Len(t, attachments, 1)
+		assert.Equal(t, attachmentID, attachments[0].Id)
+
+		// Test soft deletion
+		err = model.Instance.DeleteAttachment(attachmentID, transactionID, orgID, userID)
+		assert.NoError(t, err)
+
+		// Verify soft deletion in database
+		var deletedAttachment types.Attachment
+		err = db.Raw("SELECT deleted FROM attachment WHERE HEX(id) = ?", attachmentID).Scan(&deletedAttachment).Error
+		assert.NoError(t, err)
+		assert.True(t, deletedAttachment.Deleted)
+
+		// Verify attachment is no longer accessible
+		retrievedAttachment, err = model.Instance.GetAttachment(attachmentID, transactionID, orgID, userID)
+		assert.Error(t, err)
+		assert.Nil(t, retrievedAttachment)
+	})
+
+	t.Run("File Upload Integration Test", func(t *testing.T) {
+		// Test file upload functionality
+		filePath, originalContent := createTestFile(t)
+		defer os.Remove(filePath)
+
+		// Create upload directory
+		uploadDir := "uploads/test"
+		os.MkdirAll(uploadDir, 0755)
+		defer os.RemoveAll("uploads")
+
+		// Simulate file upload process
+		attachmentID := id.String(id.New())
+		storedFilePath := filepath.Join(uploadDir, attachmentID+".txt")
+		
+		// Copy file to upload location
+		err := copyFile(filePath, storedFilePath)
+		assert.NoError(t, err)
+
+		// Create attachment record
+		attachment := types.Attachment{
+			Id:            attachmentID,
+			TransactionId: transactionID,
+			OrgId:         orgID,
+			UserId:        userID,
+			FileName:      filepath.Base(storedFilePath),
+			OriginalName:  "test.txt",
+			ContentType:   "text/plain",
+			FileSize:      int64(len(originalContent)),
+			FilePath:      storedFilePath,
+			Description:   "Uploaded test file",
+			Uploaded:      time.Now(),
+			Deleted:       false,
+		}
+
+		createdAttachment, err := model.Instance.CreateAttachment(&attachment)
+		assert.NoError(t, err)
+		assert.NotNil(t, createdAttachment)
+
+		// Verify file exists
+		_, err = os.Stat(storedFilePath)
+		assert.NoError(t, err)
+
+		// Verify database record
+		retrievedAttachment, err := model.Instance.GetAttachment(attachmentID, transactionID, orgID, userID)
+		assert.NoError(t, err)
+		assert.Equal(t, storedFilePath, retrievedAttachment.FilePath)
+		assert.Equal(t, int64(len(originalContent)), retrievedAttachment.FileSize)
+	})
+}
+
+// Helper function to copy files
+func copyFile(src, dst string) error {
+	sourceFile, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer sourceFile.Close()
+
+	destFile, err := os.Create(dst)
+	if err != nil {
+		return err
+	}
+	defer destFile.Close()
+
+	_, err = io.Copy(destFile, sourceFile)
+	return err
+}
+
+func TestAttachmentValidation(t *testing.T) {
+	db, cleanup := setupTestDatabase(t)
+	defer cleanup()
+
+	orgID, userID, transactionID := setupTestData(t, db)
+
+	// Set up the model instance
+	bc := &util.StandardBcrypt{}
+	gormModel := model.NewGormModel(db, bc, types.Config{})
+	model.Instance = gormModel
+
+	t.Run("Invalid attachment data", func(t *testing.T) {
+		// Test with missing required fields
+		attachment := types.Attachment{
+			// Missing ID
+			TransactionId: transactionID,
+			OrgId:         orgID,
+			UserId:        userID,
+		}
+
+		createdAttachment, err := model.Instance.CreateAttachment(&attachment)
+		assert.Error(t, err)
+		assert.Nil(t, createdAttachment)
+	})
+
+	t.Run("Non-existent attachment retrieval", func(t *testing.T) {
+		nonExistentID := id.String(id.New())
+		
+		attachment, err := model.Instance.GetAttachment(nonExistentID, transactionID, orgID, userID)
+		assert.Error(t, err)
+		assert.Nil(t, attachment)
+	})
+}
--- a/core/api/attachment_storage.go
+++ b/core/api/attachment_storage.go
@@ -0,0 +1,289 @@
+package api
+
+import (
+	"fmt"
+	"io"
+	"mime/multipart"
+	"net/http"
+	"time"
+
+	"github.com/ant0ine/go-json-rest/rest"
+	"github.com/openaccounting/oa-server/core/model"
+	"github.com/openaccounting/oa-server/core/model/types"
+	"github.com/openaccounting/oa-server/core/storage"
+	"github.com/openaccounting/oa-server/core/util"
+	"github.com/openaccounting/oa-server/core/util/id"
+)
+
+// AttachmentHandler handles attachment operations with configurable storage
+type AttachmentHandler struct {
+	storage storage.Storage
+}
+
+// Global attachment handler instance (will be initialized during server startup)
+var attachmentHandler *AttachmentHandler
+
+// InitializeAttachmentHandler initializes the global attachment handler with storage backend
+func InitializeAttachmentHandler(storageConfig storage.Config) error {
+	storageBackend, err := storage.NewStorage(storageConfig)
+	if err != nil {
+		return fmt.Errorf("failed to initialize storage backend: %w", err)
+	}
+	
+	attachmentHandler = &AttachmentHandler{
+		storage: storageBackend,
+	}
+	
+	return nil
+}
+
+// PostAttachmentWithStorage handles file upload using the configured storage backend
+func PostAttachmentWithStorage(w rest.ResponseWriter, r *rest.Request) {
+	if attachmentHandler == nil {
+		rest.Error(w, "Storage backend not initialized", http.StatusInternalServerError)
+		return
+	}
+	
+	transactionId := r.FormValue("transactionId")
+	if transactionId == "" {
+		rest.Error(w, "Transaction ID is required", http.StatusBadRequest)
+		return
+	}
+
+	if !util.IsValidUUID(transactionId) {
+		rest.Error(w, "Invalid transaction ID format", http.StatusBadRequest)
+		return
+	}
+
+	user := r.Env["USER"].(*types.User)
+
+	// Parse multipart form
+	err := r.ParseMultipartForm(MaxFileSize)
+	if err != nil {
+		rest.Error(w, "Failed to parse multipart form", http.StatusBadRequest)
+		return
+	}
+
+	files := r.MultipartForm.File["file"]
+	if len(files) == 0 {
+		rest.Error(w, "No file provided", http.StatusBadRequest)
+		return
+	}
+
+	fileHeader := files[0] // Take the first file
+
+	// Verify transaction exists and user has permission
+	tx, err := model.Instance.GetTransaction(transactionId, "", user.Id)
+	if err != nil {
+		rest.Error(w, "Transaction not found", http.StatusNotFound)
+		return
+	}
+	if tx == nil {
+		rest.Error(w, "Transaction not found", http.StatusNotFound)
+		return
+	}
+
+	// Process the file upload
+	attachment, err := attachmentHandler.processFileUploadWithStorage(fileHeader, transactionId, tx.OrgId, user.Id, r.FormValue("description"))
+	if err != nil {
+		rest.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	// Save attachment to database
+	createdAttachment, err := model.Instance.CreateAttachment(attachment)
+	if err != nil {
+		// Clean up the stored file on database error
+		attachmentHandler.storage.Delete(attachment.FilePath)
+		rest.Error(w, "Failed to save attachment", http.StatusInternalServerError)
+		return
+	}
+
+	w.WriteHeader(http.StatusCreated)
+	w.WriteJson(createdAttachment)
+}
+
+// GetAttachmentWithStorage retrieves an attachment using the configured storage backend
+func GetAttachmentWithStorage(w rest.ResponseWriter, r *rest.Request) {
+	if attachmentHandler == nil {
+		rest.Error(w, "Storage backend not initialized", http.StatusInternalServerError)
+		return
+	}
+	
+	attachmentId := r.PathParam("id")
+	if !util.IsValidUUID(attachmentId) {
+		rest.Error(w, "Invalid attachment ID format", http.StatusBadRequest)
+		return
+	}
+
+	user := r.Env["USER"].(*types.User)
+
+	// Get attachment from database
+	attachment, err := model.Instance.GetAttachment(attachmentId, "", "", user.Id)
+	if err != nil {
+		rest.Error(w, "Attachment not found", http.StatusNotFound)
+		return
+	}
+
+	// Check if this is a download request
+	if r.URL.Query().Get("download") == "true" {
+		// Stream the file directly to the client
+		err := attachmentHandler.streamFile(w, attachment)
+		if err != nil {
+			rest.Error(w, "Failed to retrieve file", http.StatusInternalServerError)
+			return
+		}
+		return
+	}
+
+	// Return attachment metadata
+	w.WriteJson(attachment)
+}
+
+// GetAttachmentDownloadURL returns a download URL for an attachment
+func GetAttachmentDownloadURL(w rest.ResponseWriter, r *rest.Request) {
+	if attachmentHandler == nil {
+		rest.Error(w, "Storage backend not initialized", http.StatusInternalServerError)
+		return
+	}
+	
+	attachmentId := r.PathParam("id")
+	if !util.IsValidUUID(attachmentId) {
+		rest.Error(w, "Invalid attachment ID format", http.StatusBadRequest)
+		return
+	}
+
+	user := r.Env["USER"].(*types.User)
+
+	// Get attachment from database
+	attachment, err := model.Instance.GetAttachment(attachmentId, "", "", user.Id)
+	if err != nil {
+		rest.Error(w, "Attachment not found", http.StatusNotFound)
+		return
+	}
+
+	// Generate download URL (valid for 1 hour)
+	url, err := attachmentHandler.storage.GetURL(attachment.FilePath, time.Hour)
+	if err != nil {
+		rest.Error(w, "Failed to generate download URL", http.StatusInternalServerError)
+		return
+	}
+
+	response := map[string]string{
+		"url":       url,
+		"expiresIn": "3600", // 1 hour in seconds
+	}
+
+	w.WriteJson(response)
+}
+
+// DeleteAttachmentWithStorage deletes an attachment using the configured storage backend
+func DeleteAttachmentWithStorage(w rest.ResponseWriter, r *rest.Request) {
+	if attachmentHandler == nil {
+		rest.Error(w, "Storage backend not initialized", http.StatusInternalServerError)
+		return
+	}
+	
+	attachmentId := r.PathParam("id")
+	if !util.IsValidUUID(attachmentId) {
+		rest.Error(w, "Invalid attachment ID format", http.StatusBadRequest)
+		return
+	}
+
+	user := r.Env["USER"].(*types.User)
+
+	// Get attachment from database first
+	attachment, err := model.Instance.GetAttachment(attachmentId, "", "", user.Id)
+	if err != nil {
+		rest.Error(w, "Attachment not found", http.StatusNotFound)
+		return
+	}
+
+	// Delete from database (soft delete)
+	err = model.Instance.DeleteAttachment(attachmentId, attachment.TransactionId, attachment.OrgId, user.Id)
+	if err != nil {
+		rest.Error(w, "Failed to delete attachment", http.StatusInternalServerError)
+		return
+	}
+
+	// Delete from storage backend
+	// Note: For production, you might want to delay physical deletion
+	// and run a cleanup job later to handle any issues
+	err = attachmentHandler.storage.Delete(attachment.FilePath)
+	if err != nil {
+		// Log the error but don't fail the request since database deletion succeeded
+		// The file can be cleaned up later by a maintenance job
+		fmt.Printf("Warning: Failed to delete file from storage: %v\n", err)
+	}
+
+	w.WriteHeader(http.StatusOK)
+	w.WriteJson(map[string]string{"status": "deleted"})
+}
+
+// processFileUploadWithStorage processes a file upload using the storage backend
+func (h *AttachmentHandler) processFileUploadWithStorage(fileHeader *multipart.FileHeader, transactionId, orgId, userId, description string) (*types.Attachment, error) {
+	// Validate file size
+	if fileHeader.Size > MaxFileSize {
+		return nil, fmt.Errorf("file too large. Maximum size is %d bytes", MaxFileSize)
+	}
+
+	// Validate content type
+	contentType := fileHeader.Header.Get("Content-Type")
+	if !AllowedMimeTypes[contentType] {
+		return nil, fmt.Errorf("unsupported file type: %s", contentType)
+	}
+
+	// Open the file
+	file, err := fileHeader.Open()
+	if err != nil {
+		return nil, fmt.Errorf("failed to open uploaded file: %w", err)
+	}
+	defer file.Close()
+
+	// Store the file using the storage backend
+	storagePath, err := h.storage.Store(fileHeader.Filename, file, contentType)
+	if err != nil {
+		return nil, fmt.Errorf("failed to store file: %w", err)
+	}
+
+	// Create attachment record
+	attachment := &types.Attachment{
+		Id:            id.String(id.New()),
+		TransactionId: transactionId,
+		OrgId:         orgId,
+		UserId:        userId,
+		FileName:      storagePath, // Store the storage path/key
+		OriginalName:  fileHeader.Filename,
+		ContentType:   contentType,
+		FileSize:      fileHeader.Size,
+		FilePath:      storagePath, // For backward compatibility
+		Description:   description,
+		Uploaded:      time.Now(),
+		Deleted:       false,
+	}
+
+	return attachment, nil
+}
+
+// streamFile streams a file from storage to the HTTP response
+func (h *AttachmentHandler) streamFile(w rest.ResponseWriter, attachment *types.Attachment) error {
+	// Get file from storage
+	reader, err := h.storage.Retrieve(attachment.FilePath)
+	if err != nil {
+		return fmt.Errorf("failed to retrieve file: %w", err)
+	}
+	defer reader.Close()
+
+	// Set appropriate headers
+	w.Header().Set("Content-Type", attachment.ContentType)
+	w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", attachment.OriginalName))
+	
+	// If we know the file size, set Content-Length
+	if attachment.FileSize > 0 {
+		w.Header().Set("Content-Length", fmt.Sprintf("%d", attachment.FileSize))
+	}
+
+	// Stream the file to the client
+	_, err = io.Copy(w.(http.ResponseWriter), reader)
+	return err
+}
--- a/core/api/routes.go
+++ b/core/api/routes.go
@@ -36,6 +36,12 @@ func GetRouter(auth *AuthMiddleware, prefix string) (rest.App, error) {
 		rest.Get(prefix+"/orgs/:orgId/transactions/:transactionId/attachments/:attachmentId", auth.RequireAuth(GetAttachment)),
 		rest.Get(prefix+"/orgs/:orgId/transactions/:transactionId/attachments/:attachmentId/download", auth.RequireAuth(DownloadAttachment)),
 		rest.Delete(prefix+"/orgs/:orgId/transactions/:transactionId/attachments/:attachmentId", auth.RequireAuth(DeleteAttachment)),
+		
+		// New storage-based attachment endpoints
+		rest.Post(prefix+"/attachments", auth.RequireAuth(PostAttachmentWithStorage)),
+		rest.Get(prefix+"/attachments/:id", auth.RequireAuth(GetAttachmentWithStorage)), 
+		rest.Get(prefix+"/attachments/:id/url", auth.RequireAuth(GetAttachmentDownloadURL)),
+		rest.Delete(prefix+"/attachments/:id", auth.RequireAuth(DeleteAttachmentWithStorage)),
 		rest.Get(prefix+"/orgs/:orgId/prices", auth.RequireAuth(GetPrices)),
 		rest.Post(prefix+"/orgs/:orgId/prices", auth.RequireAuth(PostPrice)),
 		rest.Delete(prefix+"/orgs/:orgId/prices/:priceId", auth.RequireAuth(DeletePrice)),