feat: implement unified S3-compatible storage system

Consolidates storage backends into a single S3-compatible driver that supports: - AWS S3 (native) - Backblaze B2 (S3-compatible API) - Cloudflare R2 (S3-compatible API) - MinIO and other S3-compatible services - Local filesystem for development This replaces the previous separate B2 driver with a unified approach, reducing dependencies and complexity while adding support for more services. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
2025-07-01 23:07:44 +12:00
parent e3152d9f40
commit f99a866e13
14 changed files with 1650 additions and 2 deletions
--- a/core/api/attachment_integration_test.go
+++ b/core/api/attachment_integration_test.go
@@ -0,0 +1,306 @@
+package api
+
+import (
+	"io"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/openaccounting/oa-server/core/model"
+	"github.com/openaccounting/oa-server/core/model/types"
+	"github.com/openaccounting/oa-server/core/util"
+	"github.com/openaccounting/oa-server/core/util/id"
+	"github.com/openaccounting/oa-server/database"
+	"github.com/stretchr/testify/assert"
+	"gorm.io/driver/sqlite"
+	"gorm.io/gorm"
+)
+
+func setupTestDatabase(t *testing.T) (*gorm.DB, func()) {
+	// Create temporary database file
+	tmpDir := t.TempDir()
+	dbPath := filepath.Join(tmpDir, "test.db")
+	
+	db, err := gorm.Open(sqlite.Open(dbPath), &gorm.Config{})
+	if err != nil {
+		t.Fatalf("Failed to open database: %v", err)
+	}
+
+	// Set global DB for database package
+	database.DB = db
+
+	// Run migrations
+	err = database.AutoMigrate()
+	if err != nil {
+		t.Fatalf("Failed to run auto migrations: %v", err)
+	}
+
+	err = database.Migrate()
+	if err != nil {
+		t.Fatalf("Failed to run custom migrations: %v", err)
+	}
+
+	// Cleanup function
+	cleanup := func() {
+		sqlDB, _ := db.DB()
+		if sqlDB != nil {
+			sqlDB.Close()
+		}
+		os.RemoveAll(tmpDir)
+	}
+
+	return db, cleanup
+}
+
+func setupTestData(t *testing.T, db *gorm.DB) (orgID, userID, transactionID string) {
+	// Use hardcoded UUIDs without dashes for hex format
+	orgID = "550e8400e29b41d4a716446655440000"
+	userID = "550e8400e29b41d4a716446655440001"
+	transactionID = "550e8400e29b41d4a716446655440002"
+	accountID := "550e8400e29b41d4a716446655440003"
+
+	// Insert test data using raw SQL for reliability
+	now := time.Now()
+	
+	// Insert org
+	err := db.Exec("INSERT INTO orgs (id, inserted, updated, name, currency, `precision`, timezone) VALUES (UNHEX(?), ?, ?, ?, ?, ?, ?)",
+		orgID, now.UnixNano()/int64(time.Millisecond), now.UnixNano()/int64(time.Millisecond), "Test Org", "USD", 2, "UTC").Error
+	if err != nil {
+		t.Fatalf("Failed to insert org: %v", err)
+	}
+
+	// Insert user  
+	err = db.Exec("INSERT INTO users (id, inserted, updated, firstName, lastName, email, passwordHash, agreeToTerms, passwordReset, emailVerified, emailVerifyCode, signupSource) VALUES (UNHEX(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+		userID, now.UnixNano()/int64(time.Millisecond), now.UnixNano()/int64(time.Millisecond), "Test", "User", "test@example.com", "hashedpassword", true, "", true, "", "test").Error
+	if err != nil {
+		t.Fatalf("Failed to insert user: %v", err)
+	}
+
+	// Insert user-org relationship
+	err = db.Exec("INSERT INTO user_orgs (userId, orgId, admin) VALUES (UNHEX(?), UNHEX(?), ?)",
+		userID, orgID, false).Error
+	if err != nil {
+		t.Fatalf("Failed to insert user-org: %v", err)
+	}
+
+	// Insert account
+	err = db.Exec("INSERT INTO accounts (id, orgId, inserted, updated, name, parent, currency, `precision`, debitBalance) VALUES (UNHEX(?), UNHEX(?), ?, ?, ?, ?, ?, ?, ?)",
+		accountID, orgID, now.UnixNano()/int64(time.Millisecond), now.UnixNano()/int64(time.Millisecond), "Test Account", []byte{}, "USD", 2, true).Error
+	if err != nil {
+		t.Fatalf("Failed to insert account: %v", err)
+	}
+
+	// Insert transaction
+	err = db.Exec("INSERT INTO transactions (id, orgId, userId, inserted, updated, date, description, data, deleted) VALUES (UNHEX(?), UNHEX(?), UNHEX(?), ?, ?, ?, ?, ?, ?)",
+		transactionID, orgID, userID, now.UnixNano()/int64(time.Millisecond), now.UnixNano()/int64(time.Millisecond), now.UnixNano()/int64(time.Millisecond), "Test Transaction", "", false).Error
+	if err != nil {
+		t.Fatalf("Failed to insert transaction: %v", err)
+	}
+
+	// Insert split
+	err = db.Exec("INSERT INTO splits (transactionId, accountId, date, inserted, updated, amount, nativeAmount, deleted) VALUES (UNHEX(?), UNHEX(?), ?, ?, ?, ?, ?, ?)",
+		transactionID, accountID, now.UnixNano()/int64(time.Millisecond), now.UnixNano()/int64(time.Millisecond), now.UnixNano()/int64(time.Millisecond), 100, 100, false).Error
+	if err != nil {
+		t.Fatalf("Failed to insert split: %v", err)
+	}
+
+	return orgID, userID, transactionID
+}
+
+func createTestFile(t *testing.T) (string, []byte) {
+	content := []byte("This is a test file content for attachment testing")
+	tmpDir := t.TempDir()
+	filePath := filepath.Join(tmpDir, "test.txt")
+	
+	err := os.WriteFile(filePath, content, 0644)
+	if err != nil {
+		t.Fatalf("Failed to create test file: %v", err)
+	}
+	
+	return filePath, content
+}
+
+
+func TestAttachmentIntegration(t *testing.T) {
+	db, cleanup := setupTestDatabase(t)
+	defer cleanup()
+
+	orgID, userID, transactionID := setupTestData(t, db)
+
+	// Set up the model instance for the API handlers
+	bc := &util.StandardBcrypt{}
+	
+	// Use the existing datastore model which has the attachment implementation
+	// We need to create it with the database connection
+	datastoreModel := model.NewModel(nil, bc, types.Config{})
+	model.Instance = datastoreModel
+
+	t.Run("Database Integration Test", func(t *testing.T) {
+		// Test direct database operations first
+		filePath, originalContent := createTestFile(t)
+		defer os.Remove(filePath)
+
+		// Create attachment record directly
+		attachmentID := id.String(id.New())
+		uploadTime := time.Now()
+		
+		attachment := types.Attachment{
+			Id:            attachmentID,
+			TransactionId: transactionID,
+			OrgId:         orgID,
+			UserId:        userID,
+			FileName:      "stored_test.txt",
+			OriginalName:  "test.txt",
+			ContentType:   "text/plain",
+			FileSize:      int64(len(originalContent)),
+			FilePath:      "uploads/test/" + attachmentID + ".txt",
+			Description:   "Test attachment description",
+			Uploaded:      uploadTime,
+			Deleted:       false,
+		}
+
+		// Insert using the existing model
+		createdAttachment, err := model.Instance.CreateAttachment(&attachment)
+		assert.NoError(t, err)
+		assert.NotNil(t, createdAttachment)
+		assert.Equal(t, attachmentID, createdAttachment.Id)
+
+		// Verify database persistence
+		var dbAttachment types.Attachment
+		err = db.Raw("SELECT HEX(id) as id, HEX(transactionId) as transactionId, HEX(orgId) as orgId, HEX(userId) as userId, fileName, originalName, contentType, fileSize, filePath, description, uploaded, deleted FROM attachment WHERE HEX(id) = ?", attachmentID).Scan(&dbAttachment).Error
+		assert.NoError(t, err)
+		
+		assert.Equal(t, attachmentID, dbAttachment.Id)
+		assert.Equal(t, transactionID, dbAttachment.TransactionId)
+		assert.Equal(t, "Test attachment description", dbAttachment.Description)
+
+		// Test retrieval
+		retrievedAttachment, err := model.Instance.GetAttachment(attachmentID, transactionID, orgID, userID)
+		assert.NoError(t, err)
+		assert.NotNil(t, retrievedAttachment)
+		assert.Equal(t, attachmentID, retrievedAttachment.Id)
+
+		// Test listing by transaction
+		attachments, err := model.Instance.GetAttachmentsByTransaction(transactionID, orgID, userID)
+		assert.NoError(t, err)
+		assert.Len(t, attachments, 1)
+		assert.Equal(t, attachmentID, attachments[0].Id)
+
+		// Test soft deletion
+		err = model.Instance.DeleteAttachment(attachmentID, transactionID, orgID, userID)
+		assert.NoError(t, err)
+
+		// Verify soft deletion in database
+		var deletedAttachment types.Attachment
+		err = db.Raw("SELECT deleted FROM attachment WHERE HEX(id) = ?", attachmentID).Scan(&deletedAttachment).Error
+		assert.NoError(t, err)
+		assert.True(t, deletedAttachment.Deleted)
+
+		// Verify attachment is no longer accessible
+		retrievedAttachment, err = model.Instance.GetAttachment(attachmentID, transactionID, orgID, userID)
+		assert.Error(t, err)
+		assert.Nil(t, retrievedAttachment)
+	})
+
+	t.Run("File Upload Integration Test", func(t *testing.T) {
+		// Test file upload functionality
+		filePath, originalContent := createTestFile(t)
+		defer os.Remove(filePath)
+
+		// Create upload directory
+		uploadDir := "uploads/test"
+		os.MkdirAll(uploadDir, 0755)
+		defer os.RemoveAll("uploads")
+
+		// Simulate file upload process
+		attachmentID := id.String(id.New())
+		storedFilePath := filepath.Join(uploadDir, attachmentID+".txt")
+		
+		// Copy file to upload location
+		err := copyFile(filePath, storedFilePath)
+		assert.NoError(t, err)
+
+		// Create attachment record
+		attachment := types.Attachment{
+			Id:            attachmentID,
+			TransactionId: transactionID,
+			OrgId:         orgID,
+			UserId:        userID,
+			FileName:      filepath.Base(storedFilePath),
+			OriginalName:  "test.txt",
+			ContentType:   "text/plain",
+			FileSize:      int64(len(originalContent)),
+			FilePath:      storedFilePath,
+			Description:   "Uploaded test file",
+			Uploaded:      time.Now(),
+			Deleted:       false,
+		}
+
+		createdAttachment, err := model.Instance.CreateAttachment(&attachment)
+		assert.NoError(t, err)
+		assert.NotNil(t, createdAttachment)
+
+		// Verify file exists
+		_, err = os.Stat(storedFilePath)
+		assert.NoError(t, err)
+
+		// Verify database record
+		retrievedAttachment, err := model.Instance.GetAttachment(attachmentID, transactionID, orgID, userID)
+		assert.NoError(t, err)
+		assert.Equal(t, storedFilePath, retrievedAttachment.FilePath)
+		assert.Equal(t, int64(len(originalContent)), retrievedAttachment.FileSize)
+	})
+}
+
+// Helper function to copy files
+func copyFile(src, dst string) error {
+	sourceFile, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer sourceFile.Close()
+
+	destFile, err := os.Create(dst)
+	if err != nil {
+		return err
+	}
+	defer destFile.Close()
+
+	_, err = io.Copy(destFile, sourceFile)
+	return err
+}
+
+func TestAttachmentValidation(t *testing.T) {
+	db, cleanup := setupTestDatabase(t)
+	defer cleanup()
+
+	orgID, userID, transactionID := setupTestData(t, db)
+
+	// Set up the model instance
+	bc := &util.StandardBcrypt{}
+	gormModel := model.NewGormModel(db, bc, types.Config{})
+	model.Instance = gormModel
+
+	t.Run("Invalid attachment data", func(t *testing.T) {
+		// Test with missing required fields
+		attachment := types.Attachment{
+			// Missing ID
+			TransactionId: transactionID,
+			OrgId:         orgID,
+			UserId:        userID,
+		}
+
+		createdAttachment, err := model.Instance.CreateAttachment(&attachment)
+		assert.Error(t, err)
+		assert.Nil(t, createdAttachment)
+	})
+
+	t.Run("Non-existent attachment retrieval", func(t *testing.T) {
+		nonExistentID := id.String(id.New())
+		
+		attachment, err := model.Instance.GetAttachment(nonExistentID, transactionID, orgID, userID)
+		assert.Error(t, err)
+		assert.Nil(t, attachment)
+	})
+}