Ditch dbmate in favour of internal migration

such that migration files can be embedded

Author: jc21

backend/cmd/server/main.go

@@ -22,24 +22,25 @@ func main() {
 	config.Init(&version, &commit, &sentryDSN)
 	appstate := state.NewState()
 
-	setting.ApplySettings()
-	database.CheckSetup()
-
-	go worker.StartCertificateWorker(appstate)
-
-	api.StartServer()
-	irqchan := make(chan os.Signal, 1)
-	signal.Notify(irqchan, syscall.SIGINT, syscall.SIGTERM)
-
-	for irq := range irqchan {
-		if irq == syscall.SIGINT || irq == syscall.SIGTERM {
-			logger.Info("Got ", irq, " shutting server down ...")
-			// Close db
-			err := database.GetInstance().Close()
-			if err != nil {
-				logger.Error("DatabaseCloseError", err)
+	database.Migrate(func() {
+		setting.ApplySettings()
+		database.CheckSetup()
+		go worker.StartCertificateWorker(appstate)
+
+		api.StartServer()
+		irqchan := make(chan os.Signal, 1)
+		signal.Notify(irqchan, syscall.SIGINT, syscall.SIGTERM)
+
+		for irq := range irqchan {
+			if irq == syscall.SIGINT || irq == syscall.SIGTERM {
+				logger.Info("Got ", irq, " shutting server down ...")
+				// Close db
+				err := database.GetInstance().Close()
+				if err != nil {
+					logger.Error("DatabaseCloseError", err)
+				}
+				break
 			}
-			break
 		}
-	}
+	})
 }

backend/migrations/20201013035318_initial_schema.sql renamed to backend/internal/database/migrations/20201013035318_initial_schema.sql

@@ -0,0 +1,205 @@
+package database
+
+import (
+	"database/sql"
+	"embed"
+	"fmt"
+	"io/fs"
+	"path"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"npm/internal/logger"
+	"npm/internal/util"
+
+	"github.com/jmoiron/sqlx"
+)
+
+//go:embed migrations/*.sql
+var migrationFiles embed.FS
+
+// MigrationConfiguration options for the migrator.
+type MigrationConfiguration struct {
+	Table string `json:"table"`
+	mux   sync.Mutex
+}
+
+// Default migrator configuration
+var mConfiguration = MigrationConfiguration{
+	Table: "migration",
+}
+
+// ConfigureMigrator and will return error if missing required fields.
+func ConfigureMigrator(c *MigrationConfiguration) error {
+	// ensure updates to the config are atomic
+	mConfiguration.mux.Lock()
+	defer mConfiguration.mux.Unlock()
+	if c == nil {
+		return fmt.Errorf("a non nil Configuration is mandatory")
+	}
+	if strings.TrimSpace(c.Table) != "" {
+		mConfiguration.Table = c.Table
+	}
+	mConfiguration.Table = c.Table
+	return nil
+}
+
+type afterMigrationComplete func()
+
+// Migrate will perform the migration from start to finish
+func Migrate(followup afterMigrationComplete) bool {
+	logger.Info("Migration: Started")
+
+	// Try to connect to the database sleeping for 15 seconds in between
+	var db *sqlx.DB
+	for {
+		db = GetInstance()
+		if db == nil {
+			logger.Warn("Database is unavailable for migration, retrying in 15 seconds")
+			time.Sleep(15 * time.Second)
+		} else {
+			break
+		}
+	}
+
+	// Check for migration table existence
+	if !tableExists(db, mConfiguration.Table) {
+		err := createMigrationTable(db)
+		if err != nil {
+			logger.Error("MigratorError", err)
+			return false
+		}
+		logger.Info("Migration: Migration Table created")
+	}
+
+	// DO MIGRATION
+	migrationCount, migrateErr := performFileMigrations(db)
+	if migrateErr != nil {
+		logger.Error("MigratorError", migrateErr)
+	}
+
+	if migrateErr == nil {
+		logger.Info("Migration: Completed %v migration files", migrationCount)
+		followup()
+		return true
+	}
+	return false
+}
+
+// createMigrationTable performs a query to create the migration table
+// with the name specified in the configuration
+func createMigrationTable(db *sqlx.DB) error {
+	logger.Info("Migration: Creating Migration Table: %v", mConfiguration.Table)
+	// nolint:lll
+	query := fmt.Sprintf("CREATE TABLE IF NOT EXISTS `%v` (filename TEXT PRIMARY KEY, migrated_on INTEGER NOT NULL DEFAULT 0)", mConfiguration.Table)
+	_, err := db.Exec(query)
+	return err
+}
+
+// tableExists will check the database for the existence of the specified table.
+func tableExists(db *sqlx.DB, tableName string) bool {
+	query := `SELECT name FROM sqlite_master WHERE type='table' AND name = $1`
+
+	row := db.QueryRowx(query, tableName)
+	if row == nil {
+		logger.Error("MigratorError", fmt.Errorf("Cannot check if table exists, no row returned: %v", tableName))
+		return false
+	}
+
+	var exists *bool
+	if err := row.Scan(&exists); err != nil {
+		if err == sql.ErrNoRows {
+			return false
+		}
+		logger.Error("MigratorError", err)
+		return false
+	}
+	return *exists
+}
+
+// performFileMigrations will perform the actual migration,
+// importing files and updating the database with the rows imported.
+func performFileMigrations(db *sqlx.DB) (int, error) {
+	var importedCount = 0
+
+	// Grab a list of previously ran migrations from the database:
+	previousMigrations, prevErr := getPreviousMigrations(db)
+	if prevErr != nil {
+		return importedCount, prevErr
+	}
+
+	// List up the ".sql" files on disk
+	err := fs.WalkDir(migrationFiles, ".", func(file string, d fs.DirEntry, err error) error {
+		if !d.IsDir() {
+			shortFile := filepath.Base(file)
+
+			// Check if this file already exists in the previous migrations
+			// and if so, ignore it
+			if util.SliceContainsItem(previousMigrations, shortFile) {
+				return nil
+			}
+
+			logger.Info("Migration: Importing %v", shortFile)
+
+			sqlContents, ioErr := migrationFiles.ReadFile(path.Clean(file))
+			if ioErr != nil {
+				return ioErr
+			}
+
+			sqlString := string(sqlContents)
+
+			tx := db.MustBegin()
+			if _, execErr := tx.Exec(sqlString); execErr != nil {
+				return execErr
+			}
+			if commitErr := tx.Commit(); commitErr != nil {
+				return commitErr
+			}
+			if markErr := markMigrationSuccessful(db, shortFile); markErr != nil {
+				return markErr
+			}
+
+			importedCount++
+		}
+		return nil
+	})
+
+	return importedCount, err
+}
+
+// getPreviousMigrations will query the migration table for names
+// of migrations we can ignore because they should have already
+// been imported
+func getPreviousMigrations(db *sqlx.DB) ([]string, error) {
+	var existingMigrations []string
+	// nolint:gosec
+	query := fmt.Sprintf("SELECT filename FROM `%v` ORDER BY filename", mConfiguration.Table)
+	rows, err := db.Queryx(query)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			return existingMigrations, nil
+		}
+		return existingMigrations, err
+	}
+
+	for rows.Next() {
+		var filename *string
+		err := rows.Scan(&filename)
+		if err != nil {
+			return existingMigrations, err
+		}
+		existingMigrations = append(existingMigrations, *filename)
+	}
+
+	return existingMigrations, nil
+}
+
+// markMigrationSuccessful will add a row to the migration table
+func markMigrationSuccessful(db *sqlx.DB, filename string) error {
+	// nolint:gosec
+	query := fmt.Sprintf("INSERT INTO `%v` (filename) VALUES ($1)", mConfiguration.Table)
+	_, err := db.Exec(query, filename)
+	return err
+}

backend/migrations/20201013035839_initial_data.sql renamed to backend/internal/database/migrations/20201013035839_initial_data.sql

@@ -38,7 +38,6 @@ RUN mkdir -p /dist \
 FROM jc21/nginx-full:github-acme.sh AS final
 
 COPY --from=gobuild /dist/server /app/bin/server
-COPY backend/migrations /app/migrations
 
 ENV SUPPRESS_NO_CONFIG_WARNING=1
 ENV S6_FIX_ATTRS_HIDDEN=1
@@ -66,11 +65,7 @@ ARG BUILD_VERSION
 ARG BUILD_COMMIT
 ARG BUILD_DATE
 
-ENV DATABASE_URL="sqlite:////data/nginxproxymanager.db" \
-	DBMATE_MIGRATIONS_DIR="/app/migrations" \
-	DBMATE_NO_DUMP_SCHEMA="1" \
-	DBMATE_SCHEMA_FILE="/data/schema.sql" \
-	NPM_BUILD_VERSION="${BUILD_VERSION:-0.0.0}" \
+ENV NPM_BUILD_VERSION="${BUILD_VERSION:-0.0.0}" \
 	NPM_BUILD_COMMIT="${BUILD_COMMIT:-dev}" \
 	NPM_BUILD_DATE="${BUILD_DATE:-}"
 

backend/internal/database/migrator.go

@@ -10,10 +10,7 @@ ENV GOPROXY=$GOPROXY \
 	GOPRIVATE=$GOPRIVATE \
 	S6_LOGGING=0 \
 	SUPPRESS_NO_CONFIG_WARNING=1 \
-	S6_FIX_ATTRS_HIDDEN=1 \
-	DATABASE_URL="sqlite:////data/nginxproxymanager.db" \
-	DBMATE_MIGRATIONS_DIR="/app/backend/migrations" \
-	DBMATE_SCHEMA_FILE="/data/schema.sql"
+	S6_FIX_ATTRS_HIDDEN=1
 
 RUN echo "fs.file-max = 65535" > /etc/sysctl.conf