From a5ca9b4cae7625607e73996288415cfd94579616 Mon Sep 17 00:00:00 2001 From: "tmitchell@starkandwayne.com" Date: Wed, 8 Jun 2016 11:22:34 -0400 Subject: [PATCH 1/3] Fixes bug where line size had to be <64k Curse you, large Postgres rows --- plugin/postgres/plugin.go | 47 ++++++++++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 10 deletions(-) diff --git a/plugin/postgres/plugin.go b/plugin/postgres/plugin.go index 63d44c52d..01a3f3e8e 100644 --- a/plugin/postgres/plugin.go +++ b/plugin/postgres/plugin.go @@ -199,28 +199,54 @@ func (p PostgresPlugin) Restore(endpoint ShieldEndpoint) error { if err != nil { return err } - go func(out io.WriteCloser, in io.Reader) { + scanErr := make(chan error) + cmdErr := make(chan error) + go func(out io.WriteCloser, in io.Reader, errChan chan error) { DEBUG("Starting to read SQL statements from stdin...") - b := bufio.NewScanner(in) + r := bufio.NewReader(in) reg := regexp.MustCompile("^DROP DATABASE (.*);$") - i := 1 - for b.Scan() { - m := reg.FindStringSubmatch(b.Text()) + i := 0 + for { + thisLine := []byte{} + isPrefix := true + var err error + for isPrefix { + var tmpLine []byte + tmpLine, isPrefix, err = r.ReadLine() + if err != nil { + if err == io.EOF { + goto eof + } + errChan <- err + return + } + thisLine = append(thisLine, tmpLine...) + } + m := reg.FindStringSubmatch(string(thisLine)) if len(m) > 0 { DEBUG("Found dropped database '%s' on line %d", m[1], i) out.Write([]byte(fmt.Sprintf("UPDATE pg_database SET datallowconn = 'false' WHERE datname = '%s';\n", m[1]))) out.Write([]byte(fmt.Sprintf("SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '%s';\n", m[1]))) } - _, err = out.Write([]byte(b.Text() + "\n")) + _, err = out.Write([]byte(string(thisLine) + "\n")) if err != nil { - DEBUG("Scanner had an error: %s", err) + DEBUG("Error when writing to output: %s", err) + errChan <- err + return } i++ } + eof: DEBUG("Completed restore with %d lines of SQL", i) out.Close() - }(stdin, os.Stdin) - return cmd.Run() + errChan <- nil + }(stdin, os.Stdin, scanErr) + cmdErr <- cmd.Run() + err = <-scanErr + if err != nil { + return err + } + return <-cmdErr } func (p PostgresPlugin) Store(endpoint ShieldEndpoint) (string, error) { @@ -271,7 +297,8 @@ func pgConnectionInfo(endpoint ShieldEndpoint) (*PostgresConnectionInfo, error) database, err := endpoint.StringValueDefault("pg_database", "") DEBUG("PGDATABASE: '%s'", database) - bin := "/var/vcap/packages/postgres-9.4/bin" + //bin := "/var/vcap/packages/postgres-9.4/bin" + bin := "/var/vcap/packages/postgresql_9.3/bin" DEBUG("PGBINDIR: '%s'", bin) return &PostgresConnectionInfo{ From 0fb5c0face06fca67fe3119449f1af209894c152 Mon Sep 17 00:00:00 2001 From: "tmitchell@starkandwayne.com" Date: Thu, 9 Jun 2016 08:49:04 -0400 Subject: [PATCH 2/3] Channel cleanup --- plugin/postgres/plugin.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/plugin/postgres/plugin.go b/plugin/postgres/plugin.go index 01a3f3e8e..9269a32f3 100644 --- a/plugin/postgres/plugin.go +++ b/plugin/postgres/plugin.go @@ -200,8 +200,7 @@ func (p PostgresPlugin) Restore(endpoint ShieldEndpoint) error { return err } scanErr := make(chan error) - cmdErr := make(chan error) - go func(out io.WriteCloser, in io.Reader, errChan chan error) { + go func(out io.WriteCloser, in io.Reader, errChan chan<- error) { DEBUG("Starting to read SQL statements from stdin...") r := bufio.NewReader(in) reg := regexp.MustCompile("^DROP DATABASE (.*);$") @@ -241,12 +240,11 @@ func (p PostgresPlugin) Restore(endpoint ShieldEndpoint) error { out.Close() errChan <- nil }(stdin, os.Stdin, scanErr) - cmdErr <- cmd.Run() - err = <-scanErr + err = cmd.Run() if err != nil { return err } - return <-cmdErr + return <-scanErr } func (p PostgresPlugin) Store(endpoint ShieldEndpoint) (string, error) { From b3fef7a5e959504b34fb38b66820390d84d0fe82 Mon Sep 17 00:00:00 2001 From: Thomas Mitchell Date: Thu, 9 Jun 2016 10:18:57 -0400 Subject: [PATCH 3/3] Cleanup Debugging Residue --- plugin/postgres/plugin.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/plugin/postgres/plugin.go b/plugin/postgres/plugin.go index 9269a32f3..1babe58a3 100644 --- a/plugin/postgres/plugin.go +++ b/plugin/postgres/plugin.go @@ -295,8 +295,7 @@ func pgConnectionInfo(endpoint ShieldEndpoint) (*PostgresConnectionInfo, error) database, err := endpoint.StringValueDefault("pg_database", "") DEBUG("PGDATABASE: '%s'", database) - //bin := "/var/vcap/packages/postgres-9.4/bin" - bin := "/var/vcap/packages/postgresql_9.3/bin" + bin := "/var/vcap/packages/postgres-9.4/bin" DEBUG("PGBINDIR: '%s'", bin) return &PostgresConnectionInfo{