diff --git a/examples/real-project/starter/config.go b/examples/real-project/starter/config.go index 8a7fe05..149adf5 100644 --- a/examples/real-project/starter/config.go +++ b/examples/real-project/starter/config.go @@ -5,6 +5,7 @@ import ( "github.com/Liphium/magic/v2" "github.com/Liphium/magic/v2/mconfig" + "github.com/Liphium/magic/v2/mrunner/databases" "github.com/Liphium/magic/v2/scripting" ) @@ -12,8 +13,14 @@ func BuildMagicConfig() magic.Config { return magic.Config{ AppName: "magic-example-real-project", PlanDeployment: func(ctx *mconfig.Context) { - // Create a PostgreSQL database for the posts service - postsDB := ctx.NewPostgresDatabase("posts") + + // Create a new driver for PostgreSQL databases + driver := databases.NewLegacyPostgresDriver("postgres:17"). + // Create a PostgreSQL database for the posts service (the driver supports a builder pattern with this method) + NewDatabase("posts") + + // Make sure to register the driver in the context + ctx.Register(driver) // Allocate a new port for the service. This makes it possible to run multiple instances of this app // locally, without weird configuration hell. Magic will pick a port in case the preferred one is taken. @@ -22,11 +29,11 @@ func BuildMagicConfig() magic.Config { // Set up environment variables for the application ctx.WithEnvironment(mconfig.Environment{ // Database connection environment variables - "DB_HOST": postsDB.Host(ctx), - "DB_PORT": postsDB.Port(ctx), - "DB_USER": postsDB.Username(), - "DB_PASSWORD": postsDB.Password(), - "DB_DATABASE": postsDB.DatabaseName(ctx), + "DB_HOST": driver.Host(ctx), + "DB_PORT": driver.Port(ctx), + "DB_USER": driver.Username(), + "DB_PASSWORD": driver.Password(), + "DB_DATABASE": mconfig.ValueStatic("posts"), // Make the server listen on localhost using the port allocated by Magic "LISTEN": mconfig.ValueWithBase([]mconfig.EnvironmentValue{port}, func(s []string) string { @@ -41,7 +48,8 @@ func BuildMagicConfig() magic.Config { StartFunction: Start, Scripts: []scripting.Script{ // Scripts to deal with the database, can always come in handy - scripting.CreateScript("db-reset", "Reset the database by dropping and recreating all tables", ResetDatabase), + scripting.CreateScript("db-reset", "Reset the database by dropping all tables", ResetDatabase), + scripting.CreateScript("db-clear", "Clear the database by truncating all tables", ClearDatabases), scripting.CreateScript("db-seed", "Seed the database with sample posts", SeedDatabase), // Scripts to call endpoints, really useful for tests and development diff --git a/examples/real-project/starter/scripts_database.go b/examples/real-project/starter/scripts_database.go index 8b47814..a1e91f8 100644 --- a/examples/real-project/starter/scripts_database.go +++ b/examples/real-project/starter/scripts_database.go @@ -8,14 +8,31 @@ import ( "github.com/Liphium/magic/v2/mrunner" ) -// Script to reset the database by dropping and recreating all tables +// Script to clear all database tables content, but not fully delete them. +// +// Here we just use any to ignore the argument. This can be useful for scripts such as this one. +func ClearDatabases(runner *mrunner.Runner) error { + log.Println("Clearing database...") + + // Magic can clear all databases for you, don't worry, only data will be deleted meaning your schema is still all good :D + if err := runner.ClearTables(); err != nil { + log.Fatalln("Couldn't clear database tables:", err) + } + + log.Println("Database clear completed successfully!") + return nil +} + +// Script to reset the database by dropping all tables. // // Here we just use any to ignore the argument. This can be useful for scripts such as this one. func ResetDatabase(runner *mrunner.Runner) error { log.Println("Resetting database...") - // Magic can clear all databases for you, don't worry, only data will be deleted meaning your schema is still all good :D - runner.ClearDatabases() + // Magic can drop all databases for you as well, this means that all the tables are actually gone + if err := runner.DropTables(); err != nil { + log.Fatalln("Couldn't reset database tables:", err) + } log.Println("Database reset completed successfully!") return nil diff --git a/examples/real-project/starter/start_test.go b/examples/real-project/starter/start_test.go index ebc11ea..d4c954e 100644 --- a/examples/real-project/starter/start_test.go +++ b/examples/real-project/starter/start_test.go @@ -56,7 +56,7 @@ func TestApp(t *testing.T) { defer client.Close() // You can clear databases here, but if you don't rely on an empty database for a test, just not doing it is fine, too. - magic.GetTestRunner().ClearDatabases() + assert.Nil(t, magic.GetTestRunner().ClearTables()) // Yes, you can call scripts in here to make your life a little easier. if err := starter.SeedDatabase(); err != nil { diff --git a/factory.go b/factory.go index 130a08d..66d8b2c 100644 --- a/factory.go +++ b/factory.go @@ -28,7 +28,7 @@ func createFactory() (Factory, error) { return Factory{}, err } - for i := 0; i < maxRecursiveTries; i++ { + for range maxRecursiveTries { modPath := filepath.Join(dir, "go.mod") if _, err := os.Stat(modPath); err == nil { return Factory{projectDir: dir}, nil @@ -68,7 +68,7 @@ func (f *Factory) LockFile(profile string) string { // Get the location of the plan file for a profile func (f *Factory) PlanFile(profile string) string { - return filepath.Join(f.MagicDirectory(), fmt.Sprintf("%s.mplan", profile)) + return filepath.Join(f.MagicDirectory(), fmt.Sprintf("%s.json", profile)) } // Check if a profile is locked (a magic instance is running) diff --git a/initializer.go b/initializer.go index 8f32edb..ab1fc41 100644 --- a/initializer.go +++ b/initializer.go @@ -60,13 +60,6 @@ func prepare(config Config, testProfile string) (*Factory, *mrunner.Runner) { if isTestRunner { currentProfile = "test-" + testProfile } - ctx := mconfig.DefaultContext(config.AppName, currentProfile) - - // Check if all scripts should be listed - if *scriptsFlag && !isTestRunner { - listScripts(config) - return nil, nil - } // Create a factory for initializing everything factory, err := createFactory() @@ -79,6 +72,15 @@ func prepare(config Config, testProfile string) (*Factory, *mrunner.Runner) { } factory.WarnIfNotIgnored() + // Create the context for Magic config generation + ctx := mconfig.DefaultContext(config.AppName, currentProfile, factory.projectDir) + + // Check if all scripts should be listed + if *scriptsFlag && !isTestRunner { + listScripts(config) + return nil, nil + } + // Check if a script should be run script := *runFlag if script != "" && !isTestRunner { diff --git a/integration/constants.go b/integration/constants.go deleted file mode 100644 index 3d82c7e..0000000 --- a/integration/constants.go +++ /dev/null @@ -1,4 +0,0 @@ -package integration - -// The current version of Magic to make sure the CLI and generated modules are always compatible -const MagicVersion = "v1.0.0-rc12" diff --git a/integration/directory.go b/integration/directory.go deleted file mode 100644 index 42435be..0000000 --- a/integration/directory.go +++ /dev/null @@ -1,91 +0,0 @@ -package integration - -import ( - "errors" - "fmt" - "os" - "path/filepath" - "strings" -) - -// Get the magic directory (as an absolute path) -func GetMagicDirectory(amount int) (string, error) { - if amount <= 0 { - return "", errors.New("amount can't be 0 or less") - } - - wd, err := os.Getwd() - if err != nil { - return "", err - } - - for i := 0; i < amount; i++ { - - files, err := os.ReadDir(wd) - if err != nil { - return "", err - } - - foundMg := false - foundGm := false - // Find the magic folder - for _, entry := range files { - if entry.IsDir() && entry.Name() == "magic" { - foundMg = true - } else if !entry.IsDir() && entry.Name() == "go.mod" { - foundGm = true - } - } - if foundMg { - return filepath.Join(wd, "magic"), nil - } else if foundGm { - return "", fmt.Errorf("can't find magic directory, too far back, found go.mod in: %q", wd) - } - wd = filepath.Dir(wd) - } - return "", errors.New("can't find magic directory") -} - -// Check if a directory exists (argument can also just be a file) -func DoesDirExist(dirPath string) (bool, error) { - _, err := os.Stat(filepath.Dir(dirPath)) - if err != nil { - return false, fmt.Errorf("path to dir does not exist: %w", err) - } else { - s, err := os.Stat(dirPath) - if err != nil { - return true, nil - } else if !s.IsDir() { - return false, errors.New("path leads to an existing file not a dir") - } else { - return false, nil - } - } -} - -// Print all files in the current directory (useful for debugging) -func PrintCurrentDirAll() { - wd, _ := os.Getwd() - fmt.Println(wd) - files, _ := os.ReadDir(".") - - // Find the magic folder - for _, entry := range files { - fmt.Println(entry.Name()) - } -} - -// Convert a path from the go.mod file to an absolute path. -// -// For relative paths to be properly parsed you need to be in the correct directory. -func ModulePathToAbsolutePath(path string) string { - trimmed := strings.TrimSpace(path) - if strings.HasPrefix(trimmed, "./") || strings.HasPrefix(trimmed, "../") { - absolute, err := filepath.Abs(path) - if err != nil { - return path - } - return absolute - } - return path -} diff --git a/integration/execute_command.go b/integration/execute_command.go deleted file mode 100644 index 78dacd0..0000000 --- a/integration/execute_command.go +++ /dev/null @@ -1,89 +0,0 @@ -package integration - -import ( - "bufio" - "os" - "os/exec" - "path/filepath" -) - -// Build and then run a go program. -func BuildThenRun(funcPrint func(string), funcStart func(*exec.Cmd), directory string, args ...string) error { - - // Get the old working directory - workDir, err := os.Getwd() - if err != nil { - return err - } - - // Change directory to the file - if err := os.Chdir(directory); err != nil { - return err - } - - // Build the program - if err := ExecCmdWithFuncStart(funcPrint, func(c *exec.Cmd) {}, "go", "build", "-o", "program.exe"); err != nil { - return err - } - - // Change back to the original working directory - if err := os.Chdir(workDir); err != nil { - return err - } - - // Execute and return the process - if err := ExecCmdWithFuncStart(funcPrint, funcStart, filepath.Join(directory, "program.exe"), args...); err != nil { - return err - } - - return nil -} - -func ExecCmdWithFunc(funcPrint func(string), name string, args ...string) error { - cmd, err := execHelper(funcPrint, name, args...) - if err != nil { - return err - } - return cmd.Run() -} - -func ExecCmdWithFuncStart(funcPrint func(string), funcStart func(*exec.Cmd), name string, args ...string) error { - cmd, err := execHelper(funcPrint, name, args...) - if err != nil { - return err - } - if err = cmd.Start(); err != nil { - return err - } - funcStart(cmd) - return cmd.Wait() -} - -func execHelper(funcPrint func(string), name string, args ...string) (*exec.Cmd, error) { - cmd := exec.Command(name, args...) - - // Read the normal logs from the app - stdout, err := cmd.StdoutPipe() - if err != nil { - return nil, err - } - go func() { - scanner := bufio.NewScanner(stdout) - for scanner.Scan() { - funcPrint(scanner.Text()) - } - }() - - // Read the errors output from the app - stderr, err := cmd.StderrPipe() - if err != nil { - return nil, err - } - go func() { - scanner := bufio.NewScanner(stderr) - for scanner.Scan() { - funcPrint(scanner.Text()) - } - }() - return cmd, nil -} diff --git a/integration/file.go b/integration/file.go deleted file mode 100644 index 2345ba7..0000000 --- a/integration/file.go +++ /dev/null @@ -1,31 +0,0 @@ -package integration - -import ( - "io" - "os" -) - -func CopyFile(source string, destination string) error { - sourceFile, err := os.Open(source) - if err != nil { - return err - } - defer sourceFile.Close() - - destinationFile, err := os.Create(destination) - if err != nil { - return err - } - defer destinationFile.Close() - - _, err = io.Copy(destinationFile, sourceFile) - if err != nil { - return err - } - return nil -} - -// Create a new file with content. -func CreateFileWithContent(name string, content string) error { - return os.WriteFile(name, []byte(content), 0755) -} diff --git a/integration/formatting.go b/integration/formatting.go deleted file mode 100644 index bdcccb1..0000000 --- a/integration/formatting.go +++ /dev/null @@ -1,21 +0,0 @@ -package integration - -import "strings" - -func SnakeToCamelCase(s string, capitalizeFirst bool) string { - // Determine the start for the capitialization - start := 1 - if capitalizeFirst { - start = 0 - } - - // Start converting and return result - parts := strings.Split(s, "_") - for i := start; i < len(parts); i++ { - if parts[i] == "" { - continue - } - parts[i] = strings.ToUpper(parts[i][0:1]) + strings.ToLower(parts[i][1:]) - } - return strings.Join(parts, "") -} diff --git a/integration/path_evaluator.go b/integration/path_evaluator.go deleted file mode 100644 index fcf6b8d..0000000 --- a/integration/path_evaluator.go +++ /dev/null @@ -1,64 +0,0 @@ -package integration - -import ( - "errors" - "fmt" - "os" - "path/filepath" - "strings" -) - -func EvaluatePath(pta string) (dir string, filename string, path string, _ error) { - pta = strings.TrimSpace(pta) - - if isValidPathFile(pta) { - // working filepath with filename - dir, filename = filepath.Split(pta) - return dir, filename, filepath.Join(dir, filename), nil - } else if s, err := os.Stat(pta); err == nil && s.IsDir() { - // working filepath whithout filename - if lastDir := filepath.Base(pta); lastDir != "." && lastDir != "/" && lastDir != "\\" { - dir = pta - filename = lastDir + ".go" // TODO change if other fileextentions are allowed - if isValidPathFile(filepath.Join(dir, filename)) { - return dir, filename, filepath.Join(dir, filename), nil - } - } - return "", "", "", errors.New("bad path") - } else { - return "", "", "", errors.New("bad path") - } -} - -func EvaluateNewPath(pta string) (dir string, filename string, path string, _ error) { - pta = strings.TrimSpace(pta) - - // check if path is a file - if !strings.HasSuffix(pta, ".go") { - - // extend path with filename - if base := filepath.Base(pta); base != "." { - pta += ".go" - } else { - return "", "", "", errors.New("") - } - } - - if dE, err := DoesDirExist(filepath.Dir(pta)); err != nil { - return "", "", "", err - } else if dE { - return filepath.Dir(pta), filepath.Base(pta), pta, nil - } else { - if err = os.MkdirAll(filepath.Dir(pta), 0755); err != nil { - return "", "", "", fmt.Errorf("failed to create path %q: %w", filepath.Dir(pta), err) - } - return filepath.Dir(pta), filepath.Base(pta), pta, nil - } -} - -func isValidPathFile(path string) bool { - if s, err := os.Stat(path); err == nil && !s.IsDir() { - return true - } - return false -} diff --git a/integration/path_evaluator_test.go b/integration/path_evaluator_test.go deleted file mode 100644 index 86fbf22..0000000 --- a/integration/path_evaluator_test.go +++ /dev/null @@ -1,15 +0,0 @@ -package integration - -import ( - "fmt" - "testing" -) - -func TestPathEvaluator(t *testing.T) { - fmt.Println(EvaluatePath("./scripts/script1/")) - fmt.Println(EvaluatePath("./scripts/script1")) - fmt.Println(EvaluatePath("./scripts/script1.go")) - fmt.Println(EvaluatePath("./")) - fmt.Println(EvaluatePath("./scripts/script1/script7")) - fmt.Println(EvaluatePath("./scripts/script1/script7/test.go")) -} diff --git a/integration/port_scanner.go b/integration/port_scanner.go deleted file mode 100644 index 8e52a42..0000000 --- a/integration/port_scanner.go +++ /dev/null @@ -1,26 +0,0 @@ -package integration - -import ( - "fmt" - "net" -) - -// Scan a range of ports for an open port -func ScanForOpenPort(start, end uint) (uint, error) { - for port := start; port <= end; port++ { - if !ScanPort(port) { - return port, nil - } - } - return 0, fmt.Errorf("no open IPv4 port found in range %d-%d", start, end) -} - -// Scan an individual port. Returns true when the connection succeeds. -func ScanPort(port uint) bool { - conn, err := net.Dial("tcp", fmt.Sprintf(":%d", port)) - if err == nil { - conn.Close() - return true - } - return false -} diff --git a/integration/sanitize_path.go b/integration/sanitize_path.go deleted file mode 100644 index 8821c8f..0000000 --- a/integration/sanitize_path.go +++ /dev/null @@ -1,12 +0,0 @@ -package integration - -import "regexp" - -func IsPathSanitized(path string) bool { - pathRegex := "^[A-Za-z_][A-Za-z0-9_]*$" - found, err := regexp.MatchString(pathRegex, path) - if err != nil { - return false - } - return found -} diff --git a/mconfig/context.go b/mconfig/context.go index 0d5964a..a93366f 100644 --- a/mconfig/context.go +++ b/mconfig/context.go @@ -2,7 +2,6 @@ package mconfig import ( "fmt" - "log" "maps" "os" "strings" @@ -11,11 +10,11 @@ import ( type Context struct { appName string // Current app name profile string // Current profile - directory string // Current working directory + projectDir string // Current project directory environment *Environment // Environment for environment variables (can be nil) - databases []*Database + services []ServiceDriver ports []uint // All ports the user wants to allocate - plan **Plan // For later filling in with actual information + plan *Plan // For later filling in with actual information } // The app name you set in your config. @@ -40,6 +39,10 @@ func (c *Context) Ports() []uint { return c.ports } +func (c *Context) ProjectDirectory() string { + return c.projectDir +} + // Set the environment. func (c *Context) WithEnvironment(env Environment) { if c.environment == nil { @@ -78,47 +81,28 @@ func (c *Context) LoadSecretsToEnvironment(path string) error { return nil } -// Get the databases. -func (c *Context) Databases() []*Database { - return c.databases +// Get all services requested. +func (c *Context) Services() []ServiceDriver { + return c.services } // Plan for later (DO NOT EXPECT THIS TO BE FILLED BEFORE DEPLOYMENT STEP) func (c *Context) Plan() *Plan { - return *c.plan -} - -// Apply a plan for the environment in the config -func (c *Context) ApplyPlan(plan *Plan) { - *c.plan = plan -} - -func (c *Context) NewPostgresDatabase(name string) *Database { - database := &Database{ - dbType: DatabasePostgres, - name: name, - } - c.databases = append(c.databases, database) - return database + return c.plan } -// Add a new database. -func (c *Context) AddDatabase(database *Database) { - c.databases = append(c.databases, database) +// Register a service driver for a service +func (c *Context) Register(driver ServiceDriver) ServiceDriver { + c.services = append(c.services, driver) + return driver } -func DefaultContext(appName string, profile string) *Context { - workDir, err := os.Getwd() - if err != nil { - log.Fatalln("couldn't get current working directory") - } - - plan := &Plan{} +func DefaultContext(appName string, profile string, projectDir string) *Context { return &Context{ - directory: workDir, - appName: appName, - profile: profile, - databases: []*Database{}, - plan: &plan, + projectDir: projectDir, + appName: appName, + profile: profile, + services: []ServiceDriver{}, + plan: &Plan{}, } } diff --git a/mconfig/databases.go b/mconfig/databases.go deleted file mode 100644 index c08d2e6..0000000 --- a/mconfig/databases.go +++ /dev/null @@ -1,102 +0,0 @@ -package mconfig - -import ( - "fmt" -) - -type DatabaseType = uint - -const ( - DatabasePostgres DatabaseType = 1 -) - -type Database struct { - dbType DatabaseType // Type of the database - name string -} - -func (db *Database) Type() DatabaseType { - return db.dbType -} - -// Get the name of the database (as in the config) -func (db *Database) Name() string { - return db.name -} - -// Get the host of the database for environment variables -func (db *Database) Host(ctx *Context) EnvironmentValue { - return EnvironmentValue{ - get: func() string { - return ctx.Plan().Database(db.name).Hostname - }, - } -} - -// Get the name of the database for environment variables -func (db *Database) DatabaseName(ctx *Context) EnvironmentValue { - return EnvironmentValue{ - get: func() string { - return ctx.Plan().Database(db.name).Name - }, - } -} - -// Get the port of the database for environment variables -func (db *Database) Port(ctx *Context) EnvironmentValue { - return EnvironmentValue{ - get: func() string { - return fmt.Sprintf("%d", ctx.Plan().Database(db.name).Port) - }, - } -} - -// Get the password of the database for environment variables -func (db *Database) Password() EnvironmentValue { - return ValueStatic(db.DefaultPassword()) -} - -// Get the username of the database for environment variables -func (db *Database) Username() EnvironmentValue { - return ValueStatic(db.DefaultUsername()) -} - -// Get the default password for the database type -func (db *Database) DefaultPassword() string { - return DefaultPassword(db.dbType) -} - -// Get the default username for the database type -func (db *Database) DefaultUsername() string { - return DefaultUsername(db.dbType) -} - -// Get the default name for the database using the runner -func (db *Database) DefaultDatabaseName(ctx *Context) string { - return DefaultDatabaseName(ctx.profile, db.name) -} - -// Get the default password for a database by type. -func DefaultPassword(dbType DatabaseType) string { - switch dbType { - case DatabasePostgres: - return "postgres" - default: - return "admin" - } -} - -// Get the default username for a database by type. -func DefaultUsername(dbType DatabaseType) string { - switch dbType { - case DatabasePostgres: - return "postgres" - default: - return "admin" - } -} - -// Get the default database name for a database. -func DefaultDatabaseName(profile string, databaseName string) string { - return fmt.Sprintf("%s:%s", profile, databaseName) -} diff --git a/mconfig/environment.go b/mconfig/environment.go index 66e8574..d2c29d5 100644 --- a/mconfig/environment.go +++ b/mconfig/environment.go @@ -30,6 +30,10 @@ func ValueStatic(value string) EnvironmentValue { } } +func ValueFunction(get func() string) EnvironmentValue { + return EnvironmentValue{get} +} + // Create a new environment value based on other environment values. // // The index in the values array matches the output of the environment value. @@ -59,7 +63,7 @@ func (c *Context) ValuePort(preferredPort uint) EnvironmentValue { // Make sure the port isn't already allocated if slices.Contains(c.ports, preferredPort) { - log.Fatalln("port", preferredPort, "is already taken: taken ports: ", c.ports) + log.Fatalln("Port", preferredPort, "is already taken: taken ports: ", c.ports) } c.ports = append(c.ports, preferredPort) diff --git a/mconfig/plan.go b/mconfig/plan.go index d584a09..fa39ef9 100644 --- a/mconfig/plan.go +++ b/mconfig/plan.go @@ -3,40 +3,28 @@ package mconfig import ( "encoding/json" "fmt" - "log" "strings" "unicode" ) -type Plan struct { - AppName string `json:"app_name"` - Profile string `json:"profile"` - Environment map[string]string `json:"environment"` - DatabaseTypes []PlannedDatabaseType `json:"database_types"` - AllocatedPorts map[uint]uint `json:"ports"` -} - -type PlannedDatabaseType struct { - Port uint `json:"port"` - Type DatabaseType `json:"type"` - Databases []PlannedDatabase `json:"databases"` -} - -// Name for the database Docker container -func (p *PlannedDatabaseType) ContainerName(appName string, profile string) string { +// Name for a service Docker container +func ContainerName[S ServiceDriver](appName string, profile string, driver S) string { appName = EverythingToSnakeCase(appName) - return fmt.Sprintf("mgc-%s-%s-%d", appName, profile, p.Type) + return fmt.Sprintf("mgc-%s-%s-%s", appName, profile, driver.GetUniqueId()) } -type PlannedDatabase struct { - Name string `json:"name"` - Username string `json:"username"` - Password string `json:"password"` - Hostname string `json:"hostname"` +type Plan struct { + AppName string `json:"app_name"` + Profile string `json:"profile"` + Environment map[string]string `json:"environment"` + AllocatedPorts map[uint]uint `json:"ports"` + Containers map[string]ContainerAllocation `json:"containers"` // Service id -> Container allocation + Services map[string]string `json:"services"` // Service id -> Data +} - // Just for developers to access, not included in actual plan - Type DatabaseType `json:"-"` - Port uint `json:"-"` +// Name for a service container (get by plan) +func PlannedContainerName[S ServiceDriver](plan *Plan, driver S) string { + return ContainerName(plan.AppName, plan.Profile, driver) } // Turn the plan into printable form @@ -59,33 +47,6 @@ func FromPrintable(printable string) (*Plan, error) { return plan, nil } -// Get a database by its name. Panics when it can't find the database. -func (p *Plan) Database(name string) PlannedDatabase { - foundDB := PlannedDatabase{} - found := false - for _, t := range p.DatabaseTypes { - for _, db := range t.Databases { - if db.Name == name { - if found { - log.Fatalln("The database", name, "exists in the config more than once.") - } - found = true - foundDB = db - foundDB.Port = t.Port - } - } - } - if !found { - log.Fatalln("Database", name, "couldn't be found in the plan!") - } - return foundDB -} - -// Generate a connection string for the database. -func (db PlannedDatabase) ConnectString() string { - return fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=disable", db.Hostname, db.Port, db.Username, db.Password, db.Name) -} - // Convert every character except for letters and digits directly to _ func EverythingToSnakeCase(s string) string { newString := "" diff --git a/mconfig/services.go b/mconfig/services.go new file mode 100644 index 0000000..2bb650e --- /dev/null +++ b/mconfig/services.go @@ -0,0 +1,82 @@ +package mconfig + +import ( + "context" + "sync" + + "github.com/moby/moby/client" +) + +// An instruction to do something with a container. +// +// This is used by Magic to for example tell database providers to clear their databases. +type Instruction string + +const ( + InstructionDropTables Instruction = "database:drop_tables" + InstructionClearTables Instruction = "database:clear_tables" +) + +// A service driver is a manager for containers running a particular service image. +// +// That can be databases or literally anything you could imagine. It provides a unified interface for Magic to be able to properly control those Docker containers. +type ServiceDriver interface { + GetUniqueId() string + + // Should return the amount of ports required to start the container. + GetRequiredPortAmount() int + + // Should return the image. Magic will pull it automatically. + GetImage() string + + // Create a new container for this type of service + CreateContainer(ctx context.Context, c *client.Client, a ContainerAllocation) (string, error) + + // This method should check if the container with the id is healthy for this service + IsHealthy(ctx context.Context, c *client.Client, container ContainerInformation) (bool, error) + + // Called to initialize the container when it is healthy + Initialize(ctx context.Context, c *client.Client, container ContainerInformation) error + + // An instruction sent down from Magic to potentially do something with the container (not every service has to handle every instruction). + // + // When implementing, please look into the instructions you can support. + HandleInstruction(ctx context.Context, c *client.Client, container ContainerInformation, instruction Instruction) error + + // For creating a new instance of the service driver with the loaded data + Load(data string) (ServiceDriver, error) + + // Save the current data of the service driver into string form (will be persisted in the plan) + Save() (string, error) +} + +// All things required to create a service container +type ContainerAllocation struct { + Name string `json:"name"` + Ports []uint `json:"ports"` +} + +type ContainerInformation struct { + ID string `json:"id"` + Name string `json:"name"` + Ports []uint `json:"ports"` +} + +// Service registry for making sure all of the services can be created from their unique IDs (important for instruction calling outside of the main process). +// +// Service (string) -> Service Driver +var serviceRegistry *sync.Map = &sync.Map{} + +// Register a service driver for instruction calling (THIS IS NOT THE DRIVER ACTUALLY USED TO CREATE YOUR DATABASES, DO NOT USE OUTSIDE OF MAGIC INTERNALLY) +func RegisterDriver(driver ServiceDriver) { + serviceRegistry.Store(driver.GetUniqueId(), driver) +} + +// Get a service driver by its unique id (THIS IS NOT THE DRIVER ACTUALLY USED TO CREATE YOUR DATABASES, DO NOT USE OUTSIDE OF MAGIC INTERNALLY) +func GetDriver(serviceId string) (ServiceDriver, bool) { + obj, ok := serviceRegistry.Load(serviceId) + if !ok { + return nil, false + } + return obj.(ServiceDriver), true +} diff --git a/mrunner/databases/init.go b/mrunner/databases/init.go new file mode 100644 index 0000000..f1db133 --- /dev/null +++ b/mrunner/databases/init.go @@ -0,0 +1,7 @@ +package databases + +import "github.com/Liphium/magic/v2/mconfig" + +func init() { + mconfig.RegisterDriver(&PostgresDriver{}) +} diff --git a/mrunner/databases/postgres_legacy.go b/mrunner/databases/postgres_legacy.go new file mode 100644 index 0000000..6804967 --- /dev/null +++ b/mrunner/databases/postgres_legacy.go @@ -0,0 +1,121 @@ +package databases + +import ( + "encoding/json" + "fmt" + "log" + "os" + "strings" + + "github.com/Liphium/magic/v2/mconfig" + "github.com/Liphium/magic/v2/util" + _ "github.com/lib/pq" +) + +// Make sure the driver complies +var _ mconfig.ServiceDriver = &PostgresDriver{} + +// IMPORTANT: Having non-static passwords would make Magic not works as the Container allocation currently does not contain service driver data. +// +// This means that instruction calling would break if we added back password and username changing. +const ( + PostgresUsername = "postgres" + PostgresPassword = "postgres" +) + +var pgLegacyLog *log.Logger = log.New(os.Stdout, "pg-legacy ", log.Default().Flags()) + +type PostgresDriver struct { + Image string `json:"image"` + Databases []string `json:"databases"` +} + +// Create a new PostgreSQL legacy service driver. +// +// It currently supports version PostgreSQL v14-17. Use NewPostgresDriver for v18 and beyond. +// +// This driver will eventually be deprecated and replaced by the one for v18 and above. +func NewLegacyPostgresDriver(image string) *PostgresDriver { + imageVersion := strings.Split(image, ":")[1] + + // Supported (confirmed and tested) major versions for this Postgres driver + var supportedPostgresVersions = []string{"14", "15", "16", "17"} + + // Do a quick check to make sure the image version is actually supported + supported := false + for _, version := range supportedPostgresVersions { + if strings.HasPrefix(imageVersion, fmt.Sprintf("%s.", version)) || imageVersion == version { + supported = true + } + } + if !supported { + pgLegacyLog.Fatalln("ERROR: Version", imageVersion, "is currently not supported.") + } + + return &PostgresDriver{ + Image: image, + } +} + +func (pd *PostgresDriver) Load(data string) (mconfig.ServiceDriver, error) { + var driver PostgresDriver + if err := json.Unmarshal([]byte(data), &driver); err != nil { + return nil, err + } + return &driver, nil +} + +func (pd *PostgresDriver) Save() (string, error) { + bytes, err := json.Marshal(pd) + if err != nil { + return "", err + } + return string(bytes), nil +} + +func (pd *PostgresDriver) NewDatabase(name string) *PostgresDriver { + pd.Databases = append(pd.Databases, name) + return pd +} + +// A unique identifier for the database driver. This is appended to the container name to make sure we know it's the container from the driver. +func (pd *PostgresDriver) GetUniqueId() string { + return "postgres1417" // Context for this: Since this driver supports PostgreSQL v14-v17 this just makes it easier to know when seeing the container in "docker ps" or sth +} + +func (pd *PostgresDriver) GetRequiredPortAmount() int { + return 1 +} + +func (pd *PostgresDriver) GetImage() string { + return pd.Image +} + +// Get the username of the databases in this driver as a EnvironmentValue for your config. +func (pd *PostgresDriver) Username() mconfig.EnvironmentValue { + return mconfig.ValueStatic(PostgresUsername) +} + +// Get the password for the user of the databases in this driver as a EnvironmentValue for your config. +func (pd *PostgresDriver) Password() mconfig.EnvironmentValue { + return mconfig.ValueStatic(PostgresPassword) +} + +// Get hostname of the database container created by the driver as a EnvironmentValue for your config. +func (pd *PostgresDriver) Host(ctx *mconfig.Context) mconfig.EnvironmentValue { + return mconfig.ValueStatic("127.0.0.1") +} + +// Get the port of the database container created by the driver as a EnvironmentValue for your config. +func (pd *PostgresDriver) Port(ctx *mconfig.Context) mconfig.EnvironmentValue { + return mconfig.ValueFunction(func() string { + for id, container := range ctx.Plan().Containers { + if id == pd.GetUniqueId() { + return fmt.Sprintf("%d", ctx.Plan().AllocatedPorts[container.Ports[0]]) + } + } + + util.Log.Fatalln("ERROR: Couldn't find port for PostgreSQL container in plan!") + return "not found" + }) +} diff --git a/mrunner/databases/postgres_legacy_container.go b/mrunner/databases/postgres_legacy_container.go new file mode 100644 index 0000000..ba02511 --- /dev/null +++ b/mrunner/databases/postgres_legacy_container.go @@ -0,0 +1,163 @@ +package databases + +import ( + "context" + "database/sql" + "fmt" + "net/netip" + "strings" + + "github.com/Liphium/magic/v2/mconfig" + "github.com/moby/moby/api/types/container" + "github.com/moby/moby/api/types/mount" + "github.com/moby/moby/api/types/network" + "github.com/moby/moby/client" +) + +// Should create a new container for the database or use the existing one (returns container id + error in case one happened) +func (pd *PostgresDriver) CreateContainer(ctx context.Context, c *client.Client, a mconfig.ContainerAllocation) (string, error) { + + // Set to default username and password when not set + if pd.Image == "" { + return "", fmt.Errorf("please specify a proper image") + } + + // Check if the container already exists + f := make(client.Filters) + f.Add("name", a.Name) + summary, err := c.ContainerList(ctx, client.ContainerListOptions{ + Filters: f, + All: true, + }) + if err != nil { + return "", fmt.Errorf("couldn't list containers: %s", err) + } + containerId := "" + var mounts []mount.Mount = nil + for _, container := range summary.Items { + for _, n := range container.Names { + if strings.HasSuffix(n, a.Name) { + pgLegacyLog.Println("Found existing container...") + containerId = container.ID + + // Inspect the container to get the mounts + resp, err := c.ContainerInspect(ctx, container.ID, client.ContainerInspectOptions{}) + if err != nil { + return "", fmt.Errorf("couldn't inspect container: %s", err) + } + mounts = resp.Container.HostConfig.Mounts + } + } + } + + // Delete the container if it exists + if containerId != "" { + pgLegacyLog.Println("Deleting old container...") + if _, err := c.ContainerRemove(ctx, containerId, client.ContainerRemoveOptions{ + RemoveVolumes: false, + Force: true, + }); err != nil { + return "", fmt.Errorf("couldn't delete database container: %s", err) + } + } + + // Create the port on the postgres container (this is not the port for outside) + port, err := network.ParsePort("5432/tcp") + if err != nil { + return "", fmt.Errorf("couldn't create port for postgres container: %s", err) + } + exposedPorts := network.PortSet{port: struct{}{}} + + // If no existing mounts, create a new volume for PostgreSQL data + if mounts == nil { + volumeName := fmt.Sprintf("%s-data", a.Name) + mounts = []mount.Mount{ + { + Type: mount.TypeVolume, + Source: volumeName, + Target: "/var/lib/postgresql/data", + }, + } + } + + // Create the network config for the container (exposes the container to the host) + networkConf := &container.HostConfig{ + PortBindings: network.PortMap{ + port: []network.PortBinding{{HostIP: netip.MustParseAddr("127.0.0.1"), HostPort: fmt.Sprintf("%d", a.Ports[0])}}, + }, + Mounts: mounts, + } + + // Create the container + resp, err := c.ContainerCreate(ctx, client.ContainerCreateOptions{ + Config: &container.Config{ + Image: pd.Image, + Env: []string{ + fmt.Sprintf("POSTGRES_PASSWORD=%s", PostgresPassword), + fmt.Sprintf("POSTGRES_USER=%s", PostgresUsername), + "POSTGRES_DATABASE=postgres", + }, + ExposedPorts: exposedPorts, + }, + HostConfig: networkConf, + Name: a.Name, + }) + if err != nil { + return "", fmt.Errorf("couldn't create postgres container: %s", err) + } + + return resp.ID, nil +} + +// Check for postgres health +func (pd *PostgresDriver) IsHealthy(ctx context.Context, c *client.Client, container mconfig.ContainerInformation) (bool, error) { + readyCommand := "pg_isready -d postgres -U postgres -t 0" + cmd := strings.Split(readyCommand, " ") + execConfig := client.ExecCreateOptions{ + Cmd: cmd, + AttachStdout: true, + AttachStderr: true, + } + + // Try to execute the command + execIDResp, err := c.ExecCreate(ctx, container.ID, execConfig) + if err != nil { + return false, fmt.Errorf("couldn't create command for readiness of container: %s", err) + } + execStartCheck := client.ExecStartOptions{Detach: false, TTY: false} + if _, err := c.ExecStart(ctx, execIDResp.ID, execStartCheck); err != nil { + return false, fmt.Errorf("couldn't start command for readiness of container: %s", err) + } + respInspect, err := c.ExecInspect(ctx, execIDResp.ID, client.ExecInspectOptions{}) + if err != nil { + return false, fmt.Errorf("couldn't inspect command for readiness of container: %s", err) + } + + if mconfig.VerboseLogging { + pgLegacyLog.Println("Database health check response code:", respInspect.ExitCode) + } + + return respInspect.ExitCode == 0, nil +} + +// Initialize the internal container with a script (for example) +func (pd *PostgresDriver) Initialize(ctx context.Context, c *client.Client, container mconfig.ContainerInformation) error { + connStr := fmt.Sprintf("host=127.0.0.1 port=%d user=postgres password=postgres dbname=postgres sslmode=disable", container.Ports[0]) + + // Connect to the database + conn, err := sql.Open("postgres", connStr) + if err != nil { + return fmt.Errorf("couldn't connect to postgres: %s", err) + } + defer conn.Close() + + for _, db := range pd.Databases { + pgLegacyLog.Println("Creating database", db+"...") + _, err := conn.Exec(fmt.Sprintf("CREATE DATABASE %s", db)) + if err != nil && !strings.Contains(err.Error(), "already exists") { + return fmt.Errorf("couldn't create postgres database: %s", err) + } + } + + return nil +} diff --git a/mrunner/databases/postgres_legacy_instruct.go b/mrunner/databases/postgres_legacy_instruct.go new file mode 100644 index 0000000..934336a --- /dev/null +++ b/mrunner/databases/postgres_legacy_instruct.go @@ -0,0 +1,85 @@ +package databases + +import ( + "context" + "database/sql" + "fmt" + + "github.com/Liphium/magic/v2/mconfig" + "github.com/moby/moby/client" +) + +// Handles the instructions for PostgreSQL. +// Supports the following instructions currently: +// - Clear tables +// - Drop tables +func (pd *PostgresDriver) HandleInstruction(ctx context.Context, c *client.Client, container mconfig.ContainerInformation, instruction mconfig.Instruction) error { + switch instruction { + case mconfig.InstructionClearTables: + return pd.ClearTables(container) + case mconfig.InstructionDropTables: + return pd.DropTables(container) + } + return nil +} + +// iterateTablesFn is a function that processes each table in the database +type iterateTablesFn func(tableName string, conn *sql.DB) error + +// iterateTables iterates through all tables in all databases and applies the given function +func (pd *PostgresDriver) iterateTables(container mconfig.ContainerInformation, fn iterateTablesFn) error { + // For all databases, connect and iterate tables + for _, db := range pd.Databases { + if err := func() error { + connStr := fmt.Sprintf("host=127.0.0.1 port=%d user=postgres password=postgres dbname=%s sslmode=disable", container.Ports[0], db) + + // Connect to the database + conn, err := sql.Open("postgres", connStr) + if err != nil { + return fmt.Errorf("couldn't connect to postgres: %v", err) + } + defer conn.Close() + + // Get all of the tables + res, err := conn.Query("SELECT table_name FROM information_schema.tables WHERE table_schema NOT IN ('pg_catalog', 'information_schema')") + if err != nil { + return fmt.Errorf("couldn't get database tables: %v", err) + } + for res.Next() { + var name string + if err := res.Scan(&name); err != nil { + return fmt.Errorf("couldn't get database table name: %v", err) + } + if err := fn(name, conn); err != nil { + return err + } + } + + return nil + }(); err != nil { + return err + } + } + + return nil +} + +// Clear all tables in all databases (keeps table schema alive, just removes the content of all tables) +func (pd *PostgresDriver) ClearTables(container mconfig.ContainerInformation) error { + return pd.iterateTables(container, func(tableName string, conn *sql.DB) error { + if _, err := conn.Exec(fmt.Sprintf("truncate %s CASCADE", tableName)); err != nil { + return fmt.Errorf("couldn't truncate table %s: %v", tableName, err) + } + return nil + }) +} + +// Drop all tables in all databases (actually deletes all of your tables) +func (pd *PostgresDriver) DropTables(container mconfig.ContainerInformation) error { + return pd.iterateTables(container, func(tableName string, conn *sql.DB) error { + if _, err := conn.Exec(fmt.Sprintf("DROP TABLE %s CASCADE", tableName)); err != nil { + return fmt.Errorf("couldn't drop table table %s: %v", tableName, err) + } + return nil + }) +} diff --git a/mrunner/runner.go b/mrunner/runner.go index 3d5f607..eec65b4 100644 --- a/mrunner/runner.go +++ b/mrunner/runner.go @@ -9,11 +9,12 @@ const DefaultStartPort uint = 10000 const DefaultEndPort uint = 60000 type Runner struct { - appName string - profile string - client *client.Client - ctx *mconfig.Context - plan *mconfig.Plan + appName string + profile string + client *client.Client + ctx *mconfig.Context + plan *mconfig.Plan + services []mconfig.ServiceDriver } func (r *Runner) Environment() *mconfig.Environment { @@ -24,18 +25,19 @@ func (r *Runner) Environment() *mconfig.Environment { func NewRunner(ctx *mconfig.Context) (*Runner, error) { // Create a new client for the docker sdk - dc, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) + dc, err := client.New(client.FromEnv) if err != nil { return nil, err } // Create the runner return &Runner{ - appName: ctx.AppName(), - profile: ctx.Profile(), - client: dc, - ctx: ctx, - plan: ctx.Plan(), + appName: ctx.AppName(), + profile: ctx.Profile(), + client: dc, + ctx: ctx, + plan: ctx.Plan(), + services: ctx.Services(), }, nil } @@ -43,7 +45,7 @@ func NewRunner(ctx *mconfig.Context) (*Runner, error) { func NewRunnerFromPlan(plan *mconfig.Plan) (*Runner, error) { // Create a new client for the docker sdk - dc, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) + dc, err := client.New(client.FromEnv) if err != nil { return nil, err } diff --git a/mrunner/runner_deploy.go b/mrunner/runner_deploy.go index d8ea0da..b5cdab4 100644 --- a/mrunner/runner_deploy.go +++ b/mrunner/runner_deploy.go @@ -2,25 +2,22 @@ package mrunner import ( "context" - "database/sql" "fmt" - "log" - "net/netip" "os" "strings" + "sync" "time" "github.com/Liphium/magic/v2/mconfig" "github.com/Liphium/magic/v2/util" _ "github.com/lib/pq" - "github.com/moby/moby/api/types/container" "github.com/moby/moby/api/types/mount" - "github.com/moby/moby/api/types/network" "github.com/moby/moby/client" ) // Deploy all the containers nessecary for the application func (r *Runner) Deploy(deleteContainers bool) error { + ctx := context.Background() if os.Getenv("MAGIC_NO_DOCKER") == "true" { util.Log.Println("WARNING: Magic is running without Docker: This will cause databases or other services to not be started even when you specify them.") @@ -28,234 +25,243 @@ func (r *Runner) Deploy(deleteContainers bool) error { } // Make sure the Docker connection is working - _, err := r.client.Info(context.Background(), client.InfoOptions{}) + _, err := r.client.Info(ctx, client.InfoOptions{}) if client.IsErrConnectionFailed(err) { return fmt.Errorf("please make sure Docker is running, and that Magic (or the Go toolchain) has access to it. (%s)", err) } // Clear all state in case wanted if deleteContainers { - util.Log.Println("Clearing all state...") - r.Clear() + util.Log.Println("Deleting all containers and volumes...") + if err := r.DeleteEverything(); err != nil { + return fmt.Errorf("couldn't clear state: %v", err) + } } - // Deploy the database containers - for _, dbType := range r.plan.DatabaseTypes { - ctx := context.Background() - name := dbType.ContainerName(r.appName, r.profile) - util.Log.Println("Creating database container", name+"...") + // Pull all of the images in case they are not downloaded yet + if err := r.pullServiceImages(ctx); err != nil { + return err + } - // Check if the container already exists - f := make(client.Filters) - f.Add("name", name) - summary, err := r.client.ContainerList(ctx, client.ContainerListOptions{ - Filters: f, - All: true, - }) - if err != nil { - return fmt.Errorf("couldn't list containers: %s", err) - } - containerId := "" - var containerMounts []mount.Mount = nil - for _, c := range summary.Items { - for _, n := range c.Names { - if strings.Contains(n, name) { - util.Log.Println("Found existing container...") - containerId = c.ID + // Start all of the service containers + if err := r.startServiceContainers(ctx); err != nil { + return err + } - // Inspect the container to get the mounts - resp, err := r.client.ContainerInspect(ctx, c.ID, client.ContainerInspectOptions{}) - if err != nil { - return fmt.Errorf("couldn't inspect container: %s", err) - } - containerMounts = resp.Container.HostConfig.Mounts - } - } + // Load environment variables into current application + for key, value := range r.plan.Environment { + if err := os.Setenv(key, value); err != nil { + return fmt.Errorf("couldn't set environment variable %s: %s", key, err) } + } - // Delete the container if it exists - if containerId != "" { - if _, err := r.client.ContainerRemove(ctx, containerId, client.ContainerRemoveOptions{ - RemoveVolumes: false, - Force: true, - }); err != nil { - return fmt.Errorf("couldn't delete database container: %s", err) - } - } + util.Log.Println("Deployment finished.") + return nil +} + +// Pull all of the images for the services the runner has registered +func (r *Runner) pullServiceImages(ctx context.Context) error { + for _, driver := range r.services { + image := driver.GetImage() - // Create the new container with the volumes - util.Log.Println("Creating new container...") - containerId, err = r.createDatabaseContainer(ctx, dbType, name, containerMounts) + // Check if the image exists locally + _, err := r.client.ImageInspect(ctx, image) if err != nil { - return fmt.Errorf("couldn't create database container: %s", err) - } - // Start the container - util.Log.Println("Trying to start container...") - if _, err := r.client.ContainerStart(ctx, containerId, client.ContainerStartOptions{}); err != nil { - return fmt.Errorf("couldn't start postgres container: %s", err) - } + // Image not found, need to pull it + util.Log.Println("Pulling image", image+"...") - // Wait for the container to start (with pg_isready) - util.Log.Println("Waiting for PostgreSQL to be ready...") - readyCommand := "pg_isready -d postgres" - cmd := strings.Split(readyCommand, " ") - execConfig := client.ExecCreateOptions{ - Cmd: cmd, - AttachStdout: true, - AttachStderr: true, - } - for { - execIDResp, err := r.client.ExecCreate(ctx, containerId, execConfig) + reader, err := r.client.ImagePull(ctx, image, client.ImagePullOptions{}) if err != nil { - return fmt.Errorf("couldn't create command for readiness of container: %s", err) - } - execStartCheck := client.ExecStartOptions{Detach: false, TTY: false} - if _, err := r.client.ExecStart(ctx, execIDResp.ID, execStartCheck); err != nil { - return fmt.Errorf("couldn't start command for readiness of container: %s", err) - } - respInspect, err := r.client.ExecInspect(ctx, execIDResp.ID, client.ExecInspectOptions{}) - if err != nil { - return fmt.Errorf("couldn't inspect command for readiness of container: %s", err) - } - if respInspect.ExitCode == 0 { - break + return fmt.Errorf("couldn't pull image %s: %s", image, err) } + defer reader.Close() + + // Track progress with updates every second + lastUpdate := time.Now() + buf := make([]byte, 1024) + for { + n, err := reader.Read(buf) + if err != nil { + if err.Error() == "EOF" { + break + } + return fmt.Errorf("error while pulling image %s: %s", image, err) + } - time.Sleep(200 * time.Millisecond) - } - time.Sleep(200 * time.Millisecond) // Some additional time, sometimes takes longer + util.Log.Println(string(buf[:n])) - // Create all of the databases - util.Log.Println("Connecting to PostgreSQL...") - if err := r.createDatabases(dbType); err != nil { - return err - } - } + // Print progress update every second + if time.Since(lastUpdate) >= time.Second { + util.Log.Println("Downloading", image+"...") + lastUpdate = time.Now() + } - // Load environment variables into current application - util.Log.Println("Loading environment...") - for key, value := range r.plan.Environment { - if err := os.Setenv(key, value); err != nil { - return fmt.Errorf("couldn't set environment variable %s: %s", key, err) + if n == 0 { + break + } + } + + util.Log.Println("Successfully pulled image", image) } } - util.Log.Println("Deployment finished.") return nil } -// Create a new container for a postgres database. Returns container id. -func (r *Runner) createDatabaseContainer(ctx context.Context, dbType mconfig.PlannedDatabaseType, name string, mounts []mount.Mount) (string, error) { +// Create all the service containers and start them + wait until healthy and initialize +func (r *Runner) startServiceContainers(ctx context.Context) error { + var wg sync.WaitGroup + errChan := make(chan error, len(r.services)) - // Reserve the port for the container - port, err := network.ParsePort("5432/tcp") - if err != nil { - return "", fmt.Errorf("couldn't create port for postgres container: %s", err) - } - exposedPorts := network.PortSet{port: struct{}{}} - - // If no existing mounts, create a new volume for PostgreSQL data - if mounts == nil { - volumeName := fmt.Sprintf("%s-postgres-data", name) - mounts = []mount.Mount{ - { - Type: mount.TypeVolume, - Source: volumeName, - Target: "/var/lib/postgresql/data", - }, - } - } + // Deploy all service containers in parallel + for _, driver := range r.services { + wg.Add(1) + go func(driver mconfig.ServiceDriver) { + defer wg.Done() - // Create the network config for the container - networkConf := &container.HostConfig{ - PortBindings: network.PortMap{ - port: []network.PortBinding{{HostIP: netip.MustParseAddr("127.0.0.1"), HostPort: fmt.Sprintf("%d", dbType.Port)}}, - }, - Mounts: mounts, - } + name := r.plan.Containers[driver.GetUniqueId()].Name - // Check if an environment variable is set for the postgres image - postgresImage := os.Getenv("MAGIC_POSTGRES_IMAGE") - if postgresImage == "" { - postgresImage = "postgres:latest" - } + // Generate a proper port list from the allocated ports of the plan + containerPorts := []uint{} + for _, port := range r.plan.Containers[driver.GetUniqueId()].Ports { + containerPorts = append(containerPorts, r.plan.AllocatedPorts[port]) + } - // Create the container - resp, err := r.client.ContainerCreate(ctx, client.ContainerCreateOptions{ - Config: &container.Config{ - Image: postgresImage, - Env: []string{ - fmt.Sprintf("POSTGRES_PASSWORD=%s", mconfig.DefaultPassword(dbType.Type)), - fmt.Sprintf("POSTGRES_USER=%s", mconfig.DefaultUsername(dbType.Type)), - "POSTGRES_DATABASE=postgres", - }, - ExposedPorts: exposedPorts, - }, - HostConfig: networkConf, - Name: name, - }) - if err != nil { - return "", fmt.Errorf("couldn't create postgres container: %s", err) - } - return resp.ID, nil -} + // Create the container using the driver + util.Log.Println("Creating container for driver", driver.GetUniqueId()+"...") + containerID, err := driver.CreateContainer(ctx, r.client, mconfig.ContainerAllocation{ + Name: name, + Ports: containerPorts, + }) + if err != nil { + errChan <- fmt.Errorf("couldn't create container for service %s: %s", driver.GetUniqueId(), err) + return + } + + // Start the container + if _, err := r.client.ContainerStart(ctx, containerID, client.ContainerStartOptions{}); err != nil { + errChan <- fmt.Errorf("couldn't start container for service %s: %s", driver.GetUniqueId(), err) + return + } -// Connect to postgres and create all the needed databases. -func (r *Runner) createDatabases(dbType mconfig.PlannedDatabaseType) error { - connStr := fmt.Sprintf("host=127.0.0.1 port=%d user=postgres password=postgres dbname=postgres sslmode=disable", dbType.Port) + // Monitor health until the container is ready + util.Log.Println("Waiting for", name, "to be healthy...") + containerInfo := mconfig.ContainerInformation{ + ID: containerID, + Name: name, + Ports: containerPorts, + } - // Connect to the database - conn, err := sql.Open("postgres", connStr) - if err != nil { - return fmt.Errorf("couldn't connect to postgres: %s", err) + for { + healthy, err := driver.IsHealthy(ctx, r.client, containerInfo) + if err != nil { + errChan <- fmt.Errorf("couldn't check health for service %s: %s", driver.GetUniqueId(), err) + return + } + if healthy { + break + } + time.Sleep(200 * time.Millisecond) + } + time.Sleep(200 * time.Millisecond) // Some extra time, some services are a little weird with healthy state + + // Initialize the container + if err := driver.Initialize(ctx, r.client, containerInfo); err != nil { + errChan <- fmt.Errorf("couldn't initialize service %s: %s", driver.GetUniqueId(), err) + return + } + + util.Log.Println("Service", name, "is ready") + }(driver) } - defer conn.Close() - for _, db := range dbType.Databases { - util.Log.Println("Creating database", db.Name+"...") - _, err := conn.Exec(fmt.Sprintf("CREATE DATABASE %s", db.Name)) - if err != nil && !strings.Contains(err.Error(), "already exists") { - return fmt.Errorf("couldn't create postgres database: %s", err) + // Wait for all services to complete + wg.Wait() + close(errChan) + + // Check for any errors + for err := range errChan { + if err != nil { + return err } } + return nil } -// Delete all containers and reset all state -func (r *Runner) Clear() { - ctx := context.Background() - for _, dbType := range r.plan.DatabaseTypes { +// Helper function to load a driver for a driver unique id +func (r *Runner) loadDriver(id string) (mconfig.ServiceDriver, error) { + driver, ok := mconfig.GetDriver(id) + if !ok { + return nil, fmt.Errorf("couldn't find service driver for service type: %s", id) + } + + // Create a new driver from the data in the services + return driver.Load(r.Plan().Services[id]) +} + +// Helper function to iterate over containers and execute a callback for each found container +func (r *Runner) forEachContainer(ctx context.Context, callback func(service string, containerID string, container mconfig.ContainerAllocation) error) error { + for service, container := range r.plan.Containers { // Try to find the container for the type f := make(client.Filters) - name := dbType.ContainerName(r.appName, r.profile) - f.Add("name", name) + f.Add("name", container.Name) summary, err := r.client.ContainerList(ctx, client.ContainerListOptions{ Filters: f, - All: true, }) if err != nil { - log.Fatalln("Couldn't list containers:", err) + return fmt.Errorf("Couldn't list containers: %v", err) } containerId := "" for _, c := range summary.Items { for _, n := range c.Names { - if strings.Contains(n, name) { + if strings.Contains(n, container.Name) { containerId = c.ID } } } - // If there is no container, nothing to delete + // If there is no container, nothing to do if containerId == "" { continue } + // Execute the callback with the container ID, container info, and key + if err := callback(service, containerId, container); err != nil { + return err + } + } + + return nil +} + +// Stop all containers +func (r *Runner) StopContainers() error { + ctx := context.Background() + return r.forEachContainer(ctx, func(_, containerID string, container mconfig.ContainerAllocation) error { + + // Stop the container + util.Log.Println("Stopping container", container.Name+"...") + if _, err := r.client.ContainerStop(ctx, containerID, client.ContainerStopOptions{}); err != nil { + return fmt.Errorf("Couldn't stop database container: %v", err) + } + + return nil + }) +} + +// Delete all containers + their attached volumes and reset all state +func (r *Runner) DeleteEverything() error { + ctx := context.Background() + return r.forEachContainer(ctx, func(_ string, containerID string, container mconfig.ContainerAllocation) error { + // Get all the attached volumes to delete them manually - containerInfo, err := r.client.ContainerInspect(ctx, containerId, client.ContainerInspectOptions{}) + containerInfo, err := r.client.ContainerInspect(ctx, containerID, client.ContainerInspectOptions{}) if err != nil { util.Log.Println("Warning: Couldn't inspect container:", err) + return nil } var volumeNames []string if containerInfo.Container.Mounts != nil { @@ -267,12 +273,12 @@ func (r *Runner) Clear() { } // Delete the container - util.Log.Println("Deleting container", name+"...") - if _, err := r.client.ContainerRemove(ctx, containerId, client.ContainerRemoveOptions{ + util.Log.Println("Deleting container", container.Name+"...") + if _, err := r.client.ContainerRemove(ctx, containerID, client.ContainerRemoveOptions{ RemoveVolumes: false, Force: true, }); err != nil { - util.Log.Fatalln("Couldn't delete database container:", err) + return fmt.Errorf("Couldn't delete database container: %v", err) } // Delete all the attached volumes @@ -281,79 +287,64 @@ func (r *Runner) Clear() { if _, err := r.client.VolumeRemove(ctx, volumeName, client.VolumeRemoveOptions{ Force: true, }); err != nil { - util.Log.Println("Warning: Couldn't delete volume", volumeName+":", err) + return fmt.Errorf("Couldn't delete volume %s: %v", volumeName, err) } } - } + + return nil + }) } -// Stop all containers -func (r *Runner) StopContainers() { +// Clear the content of all tables from databases, at runtime +func (r *Runner) DropTables() error { ctx := context.Background() - for _, dbType := range r.plan.DatabaseTypes { - - // Try to find the container for the type - f := make(client.Filters) - name := dbType.ContainerName(r.appName, r.profile) - f.Add("name", name) - summary, err := r.client.ContainerList(ctx, client.ContainerListOptions{ - Filters: f, - }) + return r.forEachContainer(ctx, func(service, containerID string, container mconfig.ContainerAllocation) error { + driver, err := r.loadDriver(service) if err != nil { - util.Log.Fatalln("Couldn't list containers:", err) - } - containerId := "" - for _, c := range summary.Items { - for _, n := range c.Names { - if strings.Contains(n, name) { - containerId = c.ID - } - } + return err } - // If there is no container, nothing to stop - if containerId == "" { - continue + // Convert the ports + containerPorts := []uint{} + for _, port := range container.Ports { + containerPorts = append(containerPorts, r.plan.AllocatedPorts[port]) } - // Stop the container - util.Log.Println("Stopping container", name+"...") - if _, err := r.client.ContainerStop(ctx, containerId, client.ContainerStopOptions{}); err != nil { - util.Log.Fatalln("Couldn't stop database container:", err) + if err := driver.HandleInstruction(ctx, r.client, mconfig.ContainerInformation{ + ID: containerID, + Name: container.Name, + Ports: containerPorts, + }, mconfig.InstructionDropTables); err != nil { + return fmt.Errorf("couldn't drop tables: %v", err) } - } -} - -// Clear the databases, at runtime -func (r *Runner) ClearDatabases() { - // Delete all the databases of every type - for _, dbType := range r.plan.DatabaseTypes { + return nil + }) +} - for _, db := range dbType.Databases { - connStr := fmt.Sprintf("host=127.0.0.1 port=%d user=postgres password=postgres dbname=%s sslmode=disable", dbType.Port, db.Name) +// Delete all database tables from databases, at runtime +func (r *Runner) ClearTables() error { + ctx := context.Background() + return r.forEachContainer(ctx, func(service, containerID string, container mconfig.ContainerAllocation) error { + driver, err := r.loadDriver(service) + if err != nil { + return err + } - // Connect to the database - conn, err := sql.Open("postgres", connStr) - if err != nil { - log.Fatalln("couldn't connect to postgres:", err) - } - defer conn.Close() + // Convert the ports + containerPorts := []uint{} + for _, port := range container.Ports { + containerPorts = append(containerPorts, r.plan.AllocatedPorts[port]) + } - // Clear all of the tables - res, err := conn.Query("SELECT table_name FROM information_schema.tables WHERE table_schema NOT IN ('pg_catalog', 'information_schema')") - if err != nil { - log.Fatalln("couldn't get database tables:", err) - } - for res.Next() { - var name string - if err := res.Scan(&name); err != nil { - util.Log.Fatalln("couldn't get database table name:", err) - } - if _, err := conn.Exec(fmt.Sprintf("truncate %s CASCADE", name)); err != nil { - util.Log.Fatalln("couldn't delete from table", name+":", err) - } - } + if err := driver.HandleInstruction(ctx, r.client, mconfig.ContainerInformation{ + ID: containerID, + Name: container.Name, + Ports: containerPorts, + }, mconfig.InstructionClearTables); err != nil { + return fmt.Errorf("couldn't clear tables: %v", err) } - } + + return nil + }) } diff --git a/mrunner/runner_plan.go b/mrunner/runner_plan.go index 977d04a..87d9cea 100644 --- a/mrunner/runner_plan.go +++ b/mrunner/runner_plan.go @@ -1,10 +1,8 @@ package mrunner import ( - "fmt" - "maps" + "slices" - "github.com/Liphium/magic/v2/integration" "github.com/Liphium/magic/v2/mconfig" "github.com/Liphium/magic/v2/util" ) @@ -20,23 +18,48 @@ func (r *Runner) GeneratePlan() *mconfig.Plan { util.Log.Fatalln("no context set") } - // Prepare database containers - types, err := r.prepareDatabases() - if err != nil { - util.Log.Fatalln("couldn't start databases:", err) + // Set basic stuff + r.plan.AppName = r.ctx.AppName() + r.plan.Profile = r.ctx.Profile() + + // Collect all the ports that should be allocated (also for the service drivers obv) + portsToAllocate := r.ctx.Ports() + containerMap := map[string]mconfig.ContainerAllocation{} + for _, driver := range r.ctx.Services() { + if _, ok := containerMap[driver.GetUniqueId()]; ok { + util.Log.Fatalln("ERROR: You can't create multiple drivers of the same type at the moment.") + } + + alloc := mconfig.ContainerAllocation{ + Name: mconfig.PlannedContainerName(r.plan, driver), + Ports: []uint{}, + } + + for range driver.GetRequiredPortAmount() { + + // Make sure we're not allocating a port that's already taken + currentPort := util.RandomPort(DefaultStartPort, DefaultEndPort) + for slices.Contains(portsToAllocate, currentPort) && !util.ScanPort(currentPort) { + currentPort = util.RandomPort(DefaultStartPort, DefaultEndPort) + } + + // Allocate one of the default ports for the container + portsToAllocate = append(portsToAllocate, currentPort) + alloc.Ports = append(alloc.Ports, currentPort) + } + + containerMap[driver.GetUniqueId()] = alloc } // Prepare all of the ports allocatedPorts := map[uint]uint{} - if r.ctx.Ports() != nil { - for _, port := range r.ctx.Ports() { + if len(portsToAllocate) > 0 { + for _, port := range portsToAllocate { + // Generate a new port in case the current one is taken toAllocate := port - if integration.ScanPort(port) { - toAllocate, err = scanForOpenPort() - if err != nil { - util.Log.Fatalln("Couldn't find open port for", port, ":", err) - } + for !util.ScanPort(toAllocate) && slices.Contains(portsToAllocate, toAllocate) { + toAllocate = util.RandomPort(DefaultStartPort, DefaultEndPort) } // Add the port to the plan @@ -45,71 +68,24 @@ func (r *Runner) GeneratePlan() *mconfig.Plan { } // Load into plan - r.plan = &mconfig.Plan{ - AppName: r.ctx.AppName(), - Profile: r.ctx.Profile(), - DatabaseTypes: types, - AllocatedPorts: allocatedPorts, + r.plan.Containers = containerMap + r.plan.AllocatedPorts = allocatedPorts + + // Add all the services to the plan + r.plan.Services = map[string]string{} + for _, driver := range r.ctx.Services() { + data, err := driver.Save() + if err != nil { + util.Log.Fatalln("couldn't persist service driver of type", driver.GetUniqueId()+":", err) + } + r.plan.Services[driver.GetUniqueId()] = data } - r.ctx.ApplyPlan(r.plan) // Generate the environment variables and add to plan environment := map[string]string{} - if r.Environment() != nil { - environment = r.Environment().Generate() + if r.ctx.Environment() != nil { + environment = r.ctx.Environment().Generate() } r.plan.Environment = environment return r.plan } - -func (r *Runner) prepareDatabases() ([]mconfig.PlannedDatabaseType, error) { - - // Scan for open ports per type - types := map[mconfig.DatabaseType]mconfig.PlannedDatabaseType{} - for _, db := range r.ctx.Databases() { - if _, ok := types[db.Type()]; !ok { - openPort, err := scanForOpenPort() - if err != nil { - return nil, err - } - - types[db.Type()] = mconfig.PlannedDatabaseType{ - Type: db.Type(), - Port: openPort, - Databases: []mconfig.PlannedDatabase{}, - } - } - } - - // Add all of the databases - for _, db := range r.ctx.Databases() { - dbType := types[db.Type()] - dbType.Databases = append(dbType.Databases, mconfig.PlannedDatabase{ - Name: db.Name(), - Username: db.DefaultUsername(), - Password: db.DefaultPassword(), - Hostname: "127.0.0.1", - Type: dbType.Type, - Port: dbType.Port, - }) - types[db.Type()] = dbType - } - - // Convert to list - plannedTypes := make([]mconfig.PlannedDatabaseType, len(types)) - i := 0 - for value := range maps.Values(types) { - plannedTypes[i] = value - i++ - } - return plannedTypes, nil -} - -// Scan for an open port in the default range -func scanForOpenPort() (uint, error) { - openPort, err := integration.ScanForOpenPort(DefaultStartPort, DefaultEndPort) - if err != nil { - return 0, fmt.Errorf("couldn't find open port: %e", err) - } - return openPort, err -} diff --git a/util/util.go b/util/util.go index d285a13..65f8b26 100644 --- a/util/util.go +++ b/util/util.go @@ -1,8 +1,10 @@ package util import ( + "fmt" "log" "math/rand" + "net" "os" "time" ) @@ -18,3 +20,18 @@ func RandomString(length int) string { } return string(b) } + +// Generate a random port +func RandomPort(start, end uint) uint { + return start + uint(rand.Intn(int(end-start+1))) +} + +// Scan an individual port. Returns true when the creation of the listener succeeds. +func ScanPort(port uint) bool { + listener, err := net.Listen("tcp", fmt.Sprintf(":%d", port)) + if err == nil { + listener.Close() + return true + } + return false +}