From d4d997ea3378e1ec079c36865fc96534fd8aa800 Mon Sep 17 00:00:00 2001 From: lowcarbdev Date: Wed, 21 Jan 2026 00:07:51 -0700 Subject: [PATCH] default to wal mode --- .gitignore | 2 ++ docs/ADMIN.md | 24 +++++++++++++++++ internal/auth.go | 14 ++++++++++ internal/autoimport.go | 10 +++++--- internal/database.go | 58 +++++++++++++++++++++++++++++++----------- main.go | 4 +++ 6 files changed, 94 insertions(+), 18 deletions(-) diff --git a/.gitignore b/.gitignore index c0ac3c0..f3ddab3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ # Backend tmp/ *.db +*.db-shm +*.db-wal messages.db build-errors.log sbv diff --git a/docs/ADMIN.md b/docs/ADMIN.md index 3e7d8fc..017c5c6 100644 --- a/docs/ADMIN.md +++ b/docs/ADMIN.md @@ -2,6 +2,30 @@ This guide covers administrative features in SBV. +## Database Journal Mode + +SBV uses SQLite with WAL (Write-Ahead Logging) mode by default. WAL mode provides better concurrent access, allowing you to browse messages while imports are running. + +If your data directory is on a network filesystem (NFS, SMB/CIFS), WAL mode may not work correctly. In this case, use the `-journal` flag to switch to rollback journal mode: + +Docker: +```bash +docker run ... ghcr.io/lowcarbdev/sbv:stable ./sbv -journal +``` + +Binary: +```bash +./sbv -journal +``` + +Docker Compose: +```yaml +services: + sbv: + image: ghcr.io/lowcarbdev/sbv:stable + command: ["./sbv", "-journal"] +``` + ## User Management ### List All Users diff --git a/internal/auth.go b/internal/auth.go index 58c090f..ab574f2 100644 --- a/internal/auth.go +++ b/internal/auth.go @@ -25,6 +25,20 @@ func InitAuthDB(filepath string) error { return err } + // Set busy timeout for better concurrent access + _, err = authDB.Exec("PRAGMA busy_timeout=5000;") + if err != nil { + return fmt.Errorf("failed to set busy timeout: %w", err) + } + + // Enable WAL mode if requested (better for concurrent reads during writes) + if UseWALMode { + _, err = authDB.Exec("PRAGMA journal_mode=WAL;") + if err != nil { + return fmt.Errorf("failed to enable WAL mode: %w", err) + } + } + createTableSQL := ` CREATE TABLE IF NOT EXISTS users ( id TEXT PRIMARY KEY, diff --git a/internal/autoimport.go b/internal/autoimport.go index c7d7f1a..80e2ea8 100644 --- a/internal/autoimport.go +++ b/internal/autoimport.go @@ -145,6 +145,7 @@ func (s *AutoImportService) processFile(userID, filePath, filename string) { } logWriter.log("Starting import of %s", filename) + startTime := time.Now() // Get username from auth database username, err := GetUsernameByID(userID) @@ -192,10 +193,13 @@ func (s *AutoImportService) processFile(userID, filePath, filename string) { completePath = filepath.Join(completeDir, filename) } + duration := time.Since(startTime) + if parseErr != nil { logWriter.log("ERROR: Import failed: %v", parseErr) logWriter.log("File will remain in ingest directory for manual review") - slog.Error("Import failed", "userID", userID, "file", filename, "error", parseErr) + logWriter.log("Import duration: %s", duration) + slog.Error("Import failed", "userID", userID, "file", filename, "error", parseErr, "duration", duration) } else { // Move file to complete directory if err := os.Rename(filePath, completePath); err != nil { @@ -211,9 +215,9 @@ func (s *AutoImportService) processFile(userID, filePath, filename string) { slog.Warn("Failed to move log file", "userID", userID, "error", err) } - logWriter.log("Import completed successfully") + logWriter.log("Import completed successfully in %s", duration) logWriter.log("File moved to: %s", completePath) - slog.Info("Import completed", "userID", userID, "file", filename) + slog.Info("Import completed", "userID", userID, "file", filename, "duration", duration) } } diff --git a/internal/database.go b/internal/database.go index b932e5a..cfd163a 100644 --- a/internal/database.go +++ b/internal/database.go @@ -19,6 +19,9 @@ var db *sql.DB var userDBs = make(map[string]*sql.DB) var userDBsMutex sync.RWMutex +// UseWALMode controls whether WAL journal mode is enabled for databases +var UseWALMode bool + // truncateString truncates a string to maxLen characters for logging func truncateString(s string, maxLen int) string { if len(s) <= maxLen { @@ -58,6 +61,20 @@ func InitDB(filepath string) error { return err } + // Set busy timeout for better concurrent access + _, err = db.Exec("PRAGMA busy_timeout=5000;") + if err != nil { + return fmt.Errorf("failed to set busy timeout: %w", err) + } + + // Enable WAL mode if requested (better for concurrent reads during writes) + if UseWALMode { + _, err = db.Exec("PRAGMA journal_mode=WAL;") + if err != nil { + return fmt.Errorf("failed to enable WAL mode: %w", err) + } + } + createTableSQL := ` -- Unified table for SMS messages, MMS messages, and call logs -- record_type: 1 = SMS, 2 = MMS, 3 = call @@ -152,6 +169,20 @@ func InitUserDB(userID string, filepath string) error { return err } + // Set busy timeout for better concurrent access + _, err = userDB.Exec("PRAGMA busy_timeout=5000;") + if err != nil { + return fmt.Errorf("failed to set busy timeout: %w", err) + } + + // Enable WAL mode if requested (better for concurrent reads during writes) + if UseWALMode { + _, err = userDB.Exec("PRAGMA journal_mode=WAL;") + if err != nil { + return fmt.Errorf("failed to enable WAL mode: %w", err) + } + } + createTableSQL := ` -- Unified table for SMS messages, MMS messages, and call logs -- record_type: 1 = SMS, 2 = MMS, 3 = call @@ -237,32 +268,29 @@ func InitUserDB(userID string, filepath string) error { return nil } -// GetUserDB retrieves the database connection for a specific user +// GetUserDB retrieves the database connection for a specific user, creating it if it doesn't exist func GetUserDB(userID string, username string) (*sql.DB, error) { userDBsMutex.RLock() - defer userDBsMutex.RUnlock() - userDB, exists := userDBs[userID] + userDBsMutex.RUnlock() + if !exists { - // Try to open the database if it exists + // Database not in cache, try to open or create it dbPathPrefix := os.Getenv("DB_PATH_PREFIX") if dbPathPrefix == "" { dbPathPrefix = "." } // Use UUID as database filename instead of sanitized username filepath := fmt.Sprintf("%s/sbv_%s.db", dbPathPrefix, userID) - if _, err := os.Stat(filepath); err == nil { - // Database file exists, try to open it - userDBsMutex.RUnlock() - if err := InitUserDB(userID, filepath); err != nil { - userDBsMutex.RLock() - return nil, fmt.Errorf("failed to open user database: %w", err) - } - userDBsMutex.RLock() - userDB = userDBs[userID] - } else { - return nil, fmt.Errorf("user database not found for user %s", username) + + // InitUserDB will create the database if it doesn't exist + if err := InitUserDB(userID, filepath); err != nil { + return nil, fmt.Errorf("failed to initialize user database: %w", err) } + + userDBsMutex.RLock() + userDB = userDBs[userID] + userDBsMutex.RUnlock() } return userDB, nil diff --git a/main.go b/main.go index 738c782..d301580 100644 --- a/main.go +++ b/main.go @@ -23,8 +23,12 @@ func main() { // Parse CLI flags resetPassword := flag.String("reset-password", "", "Reset password for the specified username") listUsers := flag.Bool("list-users", false, "List all users") + journalMode := flag.Bool("journal", false, "Use rollback journal mode instead of WAL (for network filesystems)") flag.Parse() + // Use WAL mode by default, unless -journal flag is set + internal.UseWALMode = !*journalMode + // Initialize slog logger logger = slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{ Level: slog.LevelInfo,