diff --git a/cmd/db.go b/cmd/db.go index 409ef42380..5c625af9c3 100644 --- a/cmd/db.go +++ b/cmd/db.go @@ -238,7 +238,8 @@ var ( Use: "test [path] ...", Short: "Tests local database with pgTAP", RunE: func(cmd *cobra.Command, args []string) error { - return test.Run(cmd.Context(), args, flags.DbConfig, afero.NewOsFs()) + useShadow, _ := cmd.Flags().GetBool("use-shadow-db") + return test.Run(cmd.Context(), args, flags.DbConfig, useShadow, afero.NewOsFs()) }, } ) @@ -349,6 +350,7 @@ func init() { testFlags.String("db-url", "", "Tests the database specified by the connection string (must be percent-encoded).") testFlags.Bool("linked", false, "Runs pgTAP tests on the linked project.") testFlags.Bool("local", true, "Runs pgTAP tests on the local database.") + testFlags.Bool("use-shadow-db", false, "Creates a temporary database from migrations for running tests in isolation.") dbTestCmd.MarkFlagsMutuallyExclusive("db-url", "linked", "local") rootCmd.AddCommand(dbCmd) } diff --git a/cmd/test.go b/cmd/test.go index 55e0520664..d838743bf7 100644 --- a/cmd/test.go +++ b/cmd/test.go @@ -41,6 +41,7 @@ func init() { dbFlags.String("db-url", "", "Tests the database specified by the connection string (must be percent-encoded).") dbFlags.Bool("linked", false, "Runs pgTAP tests on the linked project.") dbFlags.Bool("local", true, "Runs pgTAP tests on the local database.") + dbFlags.Bool("use-shadow-db", false, "Creates a temporary database from migrations for running tests in isolation.") testDbCmd.MarkFlagsMutuallyExclusive("db-url", "linked", "local") testCmd.AddCommand(testDbCmd) // Build new command diff --git a/docs/supabase/test/db.md b/docs/supabase/test/db.md index 9978bb4535..b09aa66209 100644 --- a/docs/supabase/test/db.md +++ b/docs/supabase/test/db.md @@ -7,3 +7,16 @@ Requires the local development stack to be started by running `supabase start`. Runs `pg_prove` in a container with unit test files volume mounted from `supabase/tests` directory. The test file can be suffixed by either `.sql` or `.pg` extension. Since each test is wrapped in its own transaction, it will be individually rolled back regardless of success or failure. + +## Running tests against a shadow database + +Pass `--use-shadow-db` to run tests against an ephemeral shadow database instead of the local dev database. When this flag is set, the CLI: + +1. Spins up a temporary Postgres container +2. Replays all local migrations from `supabase/migrations` +3. Runs the pgTAP tests against this clean database +4. Destroys the container when finished + +Your local dev database is never touched, making this ideal for CI pipelines and ensuring tests always run against a clean, migration-defined schema. + +The shadow database uses the `shadow_port` configured in `config.toml` (default `54320`) — the same port used by `db diff`. Because they share this port, you cannot run `db diff` and `db test --use-shadow-db` simultaneously. diff --git a/docs/templates/examples.yaml b/docs/templates/examples.yaml index d0588b6fef..844203624e 100644 --- a/docs/templates/examples.yaml +++ b/docs/templates/examples.yaml @@ -295,6 +295,17 @@ supabase-test-db: All tests successful. Files=2, Tests=2, 6 wallclock secs ( 0.03 usr 0.01 sys + 0.05 cusr 0.02 csys = 0.11 CPU) Result: PASS + - id: shadow-db + name: Run tests against an isolated shadow database + code: supabase test db --use-shadow-db + response: | + Creating shadow database... + Applying migration 20220810154537_create_employees_table.sql... + /tmp/supabase/tests/nested/order_test.pg .. ok + /tmp/supabase/tests/pet_test.sql .......... ok + All tests successful. + Files=2, Tests=2, 6 wallclock secs ( 0.03 usr 0.01 sys + 0.05 cusr 0.02 csys = 0.11 CPU) + Result: PASS # TODO: use actual cli response for sso commands supabase-sso-show: - id: basic-usage diff --git a/internal/db/test/test.go b/internal/db/test/test.go index 2852ee9432..d45694cddd 100644 --- a/internal/db/test/test.go +++ b/internal/db/test/test.go @@ -16,6 +16,8 @@ import ( "github.com/jackc/pgx/v4" "github.com/spf13/afero" "github.com/spf13/viper" + "github.com/supabase/cli/internal/db/diff" + "github.com/supabase/cli/internal/db/start" "github.com/supabase/cli/internal/utils" cliConfig "github.com/supabase/cli/pkg/config" ) @@ -25,7 +27,31 @@ const ( DISABLE_PGTAP = "drop extension if exists pgtap" ) -func Run(ctx context.Context, testFiles []string, config pgconn.Config, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error { +func Run(ctx context.Context, testFiles []string, config pgconn.Config, useShadowDb bool, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error { + // Create and migrate shadow database if requested + if useShadowDb { + fmt.Fprintln(os.Stderr, "Creating shadow database for testing...") + shadow, err := diff.CreateShadowDatabase(ctx, utils.Config.Db.ShadowPort) + if err != nil { + return err + } + defer utils.DockerRemove(shadow) + if err := start.WaitForHealthyService(ctx, utils.Config.Db.HealthTimeout, shadow); err != nil { + return err + } + if err := diff.MigrateShadowDatabase(ctx, shadow, fsys, options...); err != nil { + return err + } + // Override config to point at shadow DB + config = pgconn.Config{ + Host: utils.Config.Hostname, + Port: utils.Config.Db.ShadowPort, + User: "postgres", + Password: utils.Config.Db.Password, + Database: "postgres", + } + fmt.Fprintln(os.Stderr, "Shadow database ready. Running tests...") + } // Build test command if len(testFiles) == 0 { absTestsDir, err := filepath.Abs(utils.DbTestsDir) @@ -79,7 +105,11 @@ func Run(ctx context.Context, testFiles []string, config pgconn.Config, fsys afe // Use custom network when connecting to local database // disable selinux via security-opt to allow pg-tap to work properly hostConfig := container.HostConfig{Binds: binds, SecurityOpt: []string{"label:disable"}} - if utils.IsLocalDatabase(config) { + if useShadowDb { + // Shadow container has no Docker DNS alias; use host networking + // so pg_prove reaches it via 127.0.0.1: + hostConfig.NetworkMode = network.NetworkHost + } else if utils.IsLocalDatabase(config) { config.Host = utils.DbAliases[0] config.Port = 5432 } else { diff --git a/internal/db/test/test_test.go b/internal/db/test/test_test.go index 063f9bcd11..9c81efa298 100644 --- a/internal/db/test/test_test.go +++ b/internal/db/test/test_test.go @@ -3,14 +3,20 @@ package test import ( "context" "errors" + "net/http" "testing" + "time" + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" "github.com/h2non/gock" "github.com/jackc/pgconn" "github.com/jackc/pgerrcode" + "github.com/jackc/pgx/v4" "github.com/spf13/afero" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/supabase/cli/internal/db/diff" "github.com/supabase/cli/internal/testing/apitest" "github.com/supabase/cli/internal/utils" "github.com/supabase/cli/pkg/config" @@ -44,17 +50,135 @@ func TestRunCommand(t *testing.T) { apitest.MockDockerStart(utils.Docker, utils.GetRegistryImageUrl(config.Images.PgProve), containerId) require.NoError(t, apitest.MockDockerLogs(utils.Docker, containerId, "Result: SUCCESS")) // Run test - err := Run(context.Background(), []string{"nested"}, dbConfig, fsys, conn.Intercept) + err := Run(context.Background(), []string{"nested"}, dbConfig, false, fsys, conn.Intercept) // Check error assert.NoError(t, err) }) + t.Run("runs tests with pg_prove using shadow db", func(t *testing.T) { + utils.Config.Db.MajorVersion = 14 + utils.Config.Db.ShadowPort = 54320 + utils.Config.Db.HealthTimeout = 10 * time.Second + utils.GlobalsSql = "create schema public" + utils.InitialSchemaPg14Sql = "create schema private" + // Setup in-memory fs + fsys := afero.NewMemMapFs() + require.NoError(t, utils.WriteConfig(fsys, false)) + // Setup mock postgres - two connections needed: + // conn1: MigrateShadowDatabase (GlobalsSql, InitialSchemaPg14Sql, CREATE_TEMPLATE) + // conn2: Run main path (ENABLE_PGTAP, DISABLE_PGTAP) + conn1 := pgtest.NewConn() + defer conn1.Close(t) + conn1.Query(utils.GlobalsSql). + Reply("CREATE SCHEMA"). + Query(utils.InitialSchemaPg14Sql). + Reply("CREATE SCHEMA"). + Query(diff.CREATE_TEMPLATE). + Reply("CREATE DATABASE") + conn2 := pgtest.NewConn() + defer conn2.Close(t) + conn2.Query(ENABLE_PGTAP). + Reply("CREATE EXTENSION"). + Query(DISABLE_PGTAP). + Reply("DROP EXTENSION") + // Interceptor routes by call order + called := false + interceptor := func(cc *pgx.ConnConfig) { + if !called { + called = true + conn1.Intercept(cc) + } else { + conn2.Intercept(cc) + } + } + // Setup mock docker + require.NoError(t, apitest.MockDocker(utils.Docker)) + defer gock.OffAll() + shadowId := "test-shadow-db" + apitest.MockDockerStart(utils.Docker, utils.GetRegistryImageUrl(utils.Config.Db.Image), shadowId) + gock.New(utils.Docker.DaemonHost()). + Get("/v" + utils.Docker.ClientVersion() + "/containers/" + shadowId + "/json"). + Reply(http.StatusOK). + JSON(container.InspectResponse{ContainerJSONBase: &container.ContainerJSONBase{ + State: &container.State{ + Running: true, + Health: &container.Health{Status: types.Healthy}, + }, + }}) + gock.New(utils.Docker.DaemonHost()). + Delete("/v" + utils.Docker.ClientVersion() + "/containers/" + shadowId). + Reply(http.StatusOK) + pgProveId := "test-pg-prove" + apitest.MockDockerStart(utils.Docker, utils.GetRegistryImageUrl(config.Images.PgProve), pgProveId) + require.NoError(t, apitest.MockDockerLogs(utils.Docker, pgProveId, "Result: SUCCESS")) + // Run test + err := Run(context.Background(), []string{"nested"}, dbConfig, true, fsys, interceptor) + // Check error + assert.NoError(t, err) + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + + t.Run("throws error on shadow db creation failure", func(t *testing.T) { + errNetwork := errors.New("network error") + // Setup in-memory fs + fsys := afero.NewMemMapFs() + require.NoError(t, utils.WriteConfig(fsys, false)) + // Setup mock docker + require.NoError(t, apitest.MockDocker(utils.Docker)) + defer gock.OffAll() + gock.New(utils.Docker.DaemonHost()). + Get("/v" + utils.Docker.ClientVersion() + "/images/" + utils.GetRegistryImageUrl(utils.Config.Db.Image) + "/json"). + ReplyError(errNetwork) + // Run test + err := Run(context.Background(), nil, dbConfig, true, fsys) + // Check error + assert.ErrorIs(t, err, errNetwork) + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + + t.Run("throws error on shadow db migration failure", func(t *testing.T) { + utils.Config.Db.MajorVersion = 14 + utils.Config.Db.ShadowPort = 54320 + utils.Config.Db.HealthTimeout = 10 * time.Second + utils.GlobalsSql = "create schema public" + // Setup in-memory fs + fsys := afero.NewMemMapFs() + require.NoError(t, utils.WriteConfig(fsys, false)) + // Setup mock postgres + conn := pgtest.NewConn() + defer conn.Close(t) + conn.Query(utils.GlobalsSql). + ReplyError(pgerrcode.DuplicateSchema, `schema "public" already exists`) + // Setup mock docker + require.NoError(t, apitest.MockDocker(utils.Docker)) + defer gock.OffAll() + shadowId := "test-shadow-db" + apitest.MockDockerStart(utils.Docker, utils.GetRegistryImageUrl(utils.Config.Db.Image), shadowId) + gock.New(utils.Docker.DaemonHost()). + Get("/v" + utils.Docker.ClientVersion() + "/containers/" + shadowId + "/json"). + Reply(http.StatusOK). + JSON(container.InspectResponse{ContainerJSONBase: &container.ContainerJSONBase{ + State: &container.State{ + Running: true, + Health: &container.Health{Status: types.Healthy}, + }, + }}) + gock.New(utils.Docker.DaemonHost()). + Delete("/v" + utils.Docker.ClientVersion() + "/containers/" + shadowId). + Reply(http.StatusOK) + // Run test + err := Run(context.Background(), nil, dbConfig, true, fsys, conn.Intercept) + // Check error + assert.ErrorContains(t, err, `ERROR: schema "public" already exists (SQLSTATE 42P06)`) + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + t.Run("throws error on connect failure", func(t *testing.T) { // Setup in-memory fs fsys := afero.NewMemMapFs() require.NoError(t, utils.WriteConfig(fsys, false)) // Run test - err := Run(context.Background(), nil, dbConfig, fsys) + err := Run(context.Background(), nil, dbConfig, false, fsys) // Check error assert.ErrorContains(t, err, "failed to connect to postgres") }) @@ -69,7 +193,7 @@ func TestRunCommand(t *testing.T) { conn.Query(ENABLE_PGTAP). ReplyError(pgerrcode.DuplicateObject, `extension "pgtap" already exists, skipping`) // Run test - err := Run(context.Background(), nil, dbConfig, fsys, conn.Intercept) + err := Run(context.Background(), nil, dbConfig, false, fsys, conn.Intercept) // Check error assert.ErrorContains(t, err, "failed to enable pgTAP") }) @@ -93,7 +217,7 @@ func TestRunCommand(t *testing.T) { Get("/v" + utils.Docker.ClientVersion() + "/images/" + utils.GetRegistryImageUrl(config.Images.PgProve) + "/json"). ReplyError(errNetwork) // Run test - err := Run(context.Background(), nil, dbConfig, fsys, conn.Intercept) + err := Run(context.Background(), nil, dbConfig, false, fsys, conn.Intercept) // Check error assert.ErrorIs(t, err, errNetwork) assert.Empty(t, apitest.ListUnmatchedRequests()) diff --git a/pkg/config/templates/config.toml b/pkg/config/templates/config.toml index 44b58cdb0c..cf1ccb3a80 100644 --- a/pkg/config/templates/config.toml +++ b/pkg/config/templates/config.toml @@ -27,7 +27,7 @@ enabled = false [db] # Port to use for the local database URL. port = 54322 -# Port used by db diff command to initialize the shadow database. +# Port used by db diff and test db commands to initialize the shadow database. shadow_port = 54320 # Maximum amount of time to wait for health check when starting the local database. health_timeout = "2m" diff --git a/pkg/config/testdata/config.toml b/pkg/config/testdata/config.toml index b228a9c073..69da8804d8 100644 --- a/pkg/config/testdata/config.toml +++ b/pkg/config/testdata/config.toml @@ -27,7 +27,7 @@ key_path = "../certs/my-key.pem" [db] # Port to use for the local database URL. port = 54322 -# Port used by db diff command to initialize the shadow database. +# Port used by db diff and test db commands to initialize the shadow database. shadow_port = 54320 # Maximum amount of time to wait for health check when starting the local database. health_timeout = "2m"