Skip to content

feat(spanner): add support for proto bundle DDL statements #1235

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

feat(spanner): add support for proto bundle DDL statements #1235

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions database/spanner/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ as described in [README.md#database-urls](../../README.md#database-urls)
| ----- | ------------------- | ----------- |
| `x-migrations-table` | `MigrationsTable` | Name of the migrations table |
| `x-clean-statements` | `CleanStatements` | Whether to parse and clean DDL statements before running migration towards Spanner (Required for comments and multiple statements) |
| `x-proto-descriptors-file` | `ProtoDescriptorsFile` | File containing proto descriptors to use for `PROTO BUNDLE` statements |
| `url` | `DatabaseName` | The full path to the Spanner database resource. If provided as part of `Config` it must not contain a scheme or query string to match the format `projects/{projectId}/instances/{instanceId}/databases/{databaseName}`|
| `projectId` || The Google Cloud Platform project id
| `instanceId` || The id of the instance running Spanner
Expand Down
26 changes: 20 additions & 6 deletions database/spanner/spanner.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"io"
"log"
nurl "net/url"
"os"
"regexp"
"strconv"
"strings"
Expand Down Expand Up @@ -55,7 +56,8 @@ type Config struct {
// running them towards Spanner.
// Parsing outputs clean DDL statements such as reformatted
// and void of comments.
CleanStatements bool
CleanStatements bool
ProtoDescriptorsFile string
}

// Spanner implements database.Driver for Google Cloud Spanner
Expand Down Expand Up @@ -136,11 +138,14 @@ func (s *Spanner) Open(url string) (database.Driver, error) {
}
}

protoDescriptorsFile := purl.Query().Get("x-proto-descriptors-file")

db := &DB{admin: adminClient, data: dataClient}
return WithInstance(db, &Config{
DatabaseName: dbname,
MigrationsTable: migrationsTable,
CleanStatements: clean,
DatabaseName: dbname,
MigrationsTable: migrationsTable,
CleanStatements: clean,
ProtoDescriptorsFile: protoDescriptorsFile,
})
}

Expand Down Expand Up @@ -182,10 +187,19 @@ func (s *Spanner) Run(migration io.Reader) error {
}
}

var protoDescriptors []byte
if s.config.ProtoDescriptorsFile != "" {
protoDescriptors, err = os.ReadFile(s.config.ProtoDescriptorsFile)
if err != nil {
return err
}
}

ctx := context.Background()
op, err := s.db.admin.UpdateDatabaseDdl(ctx, &adminpb.UpdateDatabaseDdlRequest{
Database: s.config.DatabaseName,
Statements: stmts,
Database: s.config.DatabaseName,
Statements: stmts,
ProtoDescriptors: protoDescriptors,
})

if err != nil {
Expand Down
91 changes: 47 additions & 44 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
module github.com/golang-migrate/migrate/v4

go 1.22.0
go 1.23.0

toolchain go1.23.1

require (
cloud.google.com/go/spanner v1.56.0
cloud.google.com/go/storage v1.38.0
cloud.google.com/go/spanner v1.76.1
cloud.google.com/go/storage v1.50.0
github.com/Azure/go-autorest/autorest/adal v0.9.16
github.com/ClickHouse/clickhouse-go v1.4.3
github.com/aws/aws-sdk-go v1.49.6
Expand All @@ -33,47 +33,52 @@ require (
github.com/nakagami/firebirdsql v0.0.0-20190310045651-3c02a58cfed8
github.com/neo4j/neo4j-go-driver v1.8.1-0.20200803113522-b626aa943eba
github.com/snowflakedb/gosnowflake v1.6.19
github.com/stretchr/testify v1.9.0
github.com/stretchr/testify v1.10.0
github.com/xanzy/go-gitlab v0.15.0
go.mongodb.org/mongo-driver v1.7.5
go.uber.org/atomic v1.7.0
golang.org/x/oauth2 v0.18.0
golang.org/x/oauth2 v0.26.0
golang.org/x/tools v0.24.0
google.golang.org/api v0.169.0
google.golang.org/api v0.222.0
modernc.org/ql v1.0.0
modernc.org/sqlite v1.18.1
)

require (
cel.dev/expr v0.19.0 // indirect
cloud.google.com/go/auth v0.14.1 // indirect
cloud.google.com/go/auth/oauth2adapt v0.2.7 // indirect
cloud.google.com/go/monitoring v1.24.0 // indirect
github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp v1.5.2 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.49.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.49.0 // indirect
github.com/distribution/reference v0.6.0 // indirect
github.com/docker/go-connections v0.5.0 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/jackc/puddle/v2 v2.2.1 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/moby/docker-image-spec v1.3.1 // indirect
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/rogpeppe/go-internal v1.12.0 // indirect
github.com/stretchr/objx v0.5.2 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect
go.opentelemetry.io/otel v1.29.0 // indirect
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
go.opentelemetry.io/contrib/detectors/gcp v1.34.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.58.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 // indirect
go.opentelemetry.io/otel v1.34.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.29.0 // indirect
go.opentelemetry.io/otel/metric v1.29.0 // indirect
go.opentelemetry.io/otel/sdk v1.29.0 // indirect
go.opentelemetry.io/otel/trace v1.29.0 // indirect
go.opentelemetry.io/otel/metric v1.34.0 // indirect
go.opentelemetry.io/otel/sdk v1.34.0 // indirect
go.opentelemetry.io/otel/sdk/metric v1.32.0 // indirect
go.opentelemetry.io/otel/trace v1.34.0 // indirect
go.opentelemetry.io/proto/otlp v1.3.1 // indirect
)

require (
cloud.google.com/go v0.112.1 // indirect
cloud.google.com/go/compute v1.25.1 // indirect
cloud.google.com/go/compute/metadata v0.2.3 // indirect
cloud.google.com/go/iam v1.1.6 // indirect
cloud.google.com/go/longrunning v0.5.5 // indirect
cloud.google.com/go v0.118.2 // indirect
cloud.google.com/go/compute/metadata v0.6.0 // indirect
cloud.google.com/go/iam v1.4.0 // indirect
cloud.google.com/go/longrunning v0.6.4 // indirect
github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect
github.com/99designs/keyring v1.2.1 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0 // indirect
Expand Down Expand Up @@ -102,17 +107,17 @@ require (
github.com/aws/aws-sdk-go-v2/service/s3 v1.27.11 // indirect
github.com/aws/smithy-go v1.13.3 // indirect
github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 // indirect
github.com/cncf/xds/go v0.0.0-20240318125728-8a4994d93e50 // indirect
github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 // indirect
github.com/cznic/mathutil v0.0.0-20180504122225-ca4c9f2c1369 // indirect
github.com/danieljoos/wincred v1.1.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/dvsekhvalnov/jose2go v1.6.0 // indirect
github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712 // indirect
github.com/envoyproxy/go-control-plane v0.12.0 // indirect
github.com/envoyproxy/protoc-gen-validate v1.0.4 // indirect
github.com/envoyproxy/go-control-plane v0.13.1 // indirect
github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect
github.com/form3tech-oss/jwt-go v3.2.5+incompatible // indirect
github.com/gabriel-vasile/mimetype v1.4.1 // indirect
github.com/go-stack/stack v1.8.0 // indirect
Expand All @@ -123,14 +128,13 @@ require (
github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect
github.com/golang-sql/sqlexp v0.1.0 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/flatbuffers v2.0.8+incompatible // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/google/s2a-go v0.1.7 // indirect
github.com/google/s2a-go v0.1.9 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
github.com/googleapis/gax-go/v2 v2.12.2 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect
github.com/googleapis/gax-go/v2 v2.14.1 // indirect
github.com/gorilla/handlers v1.4.2 // indirect
github.com/gorilla/mux v1.7.4 // indirect
github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect
Expand Down Expand Up @@ -176,22 +180,21 @@ require (
github.com/zeebo/xxh3 v1.0.2 // indirect
gitlab.com/nyarla/go-crypt v0.0.0-20160106005555-d9a5dc2b789b // indirect
go.opencensus.io v0.24.0 // indirect
golang.org/x/crypto v0.31.0 // indirect
golang.org/x/crypto v0.33.0 // indirect
golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0 // indirect
golang.org/x/mod v0.21.0 // indirect
golang.org/x/net v0.33.0 // indirect
golang.org/x/sync v0.10.0 // indirect
golang.org/x/sys v0.28.0 // indirect
golang.org/x/term v0.27.0 // indirect
golang.org/x/text v0.21.0 // indirect
golang.org/x/time v0.5.0 // indirect
golang.org/x/net v0.35.0 // indirect
golang.org/x/sync v0.11.0 // indirect
golang.org/x/sys v0.30.0 // indirect
golang.org/x/term v0.29.0 // indirect
golang.org/x/text v0.22.0 // indirect
golang.org/x/time v0.10.0 // indirect
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
google.golang.org/appengine v1.6.8 // indirect
google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240513163218-0867130af1f8 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240513163218-0867130af1f8 // indirect
google.golang.org/grpc v1.64.1 // indirect
google.golang.org/protobuf v1.34.2 // indirect
google.golang.org/genproto v0.0.0-20250122153221-138b5a5a4fd4 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20250219182151-9fdb1cabc7b2 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20250219182151-9fdb1cabc7b2 // indirect
google.golang.org/grpc v1.70.0 // indirect
google.golang.org/protobuf v1.36.5 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
lukechampine.com/uint128 v1.2.0 // indirect
Expand Down
Loading
Loading