Merge pull request #7 from ngtuna/cobra

refactor: add cobra

cmd/chart-repo/Dockerfile

@@ -0,0 +1,10 @@
+FROM quay.io/deis/go-dev:v1.6.0 as builder
+COPY . /go/src/github.com/kubeapps/chartsvc/cmd/chart-repo
+WORKDIR /go/src/github.com/kubeapps/chartsvc/cmd/chart-repo
+RUN dep ensure
+RUN CGO_ENABLED=0 go build -a -installsuffix cgo
+
+FROM scratch
+COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
+COPY --from=builder /go/src/github.com/kubeapps/chartsvc/cmd/chart-repo/chart-repo /chart-repo
+CMD ["/chart-repo"]

cmd/chart-repo-sync/Gopkg.toml → cmd/chart-repo/Gopkg.toml

@@ -44,3 +44,6 @@
 [[constraint]]
   name = "k8s.io/helm"
   version = "2.7.2"
+[[constraint]]
+  name = "github.com/spf13/cobra"
+  version = "0.0.1"

cmd/chart-repo/chart_repo.go

@@ -0,0 +1,50 @@
+/*
+Copyright (c) 2017-2018 Bitnami
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"os"
+
+	"github.com/spf13/cobra"
+)
+
+var RootCmd = &cobra.Command{
+	Use:   "chart-repo",
+	Short: "Kubeapps Chart Repository utility",
+	Run: func(cmd *cobra.Command, args []string) {
+		cmd.Help()
+	},
+}
+
+func main() {
+	cmd := RootCmd
+	if err := cmd.Execute(); err != nil {
+		os.Exit(1)
+	}
+}
+
+func init() {
+	cmds := []*cobra.Command{syncCmd, deleteCmd}
+
+	for _, cmd := range cmds {
+		RootCmd.AddCommand(cmd)
+		cmd.Flags().String("mongo-url", "localhost", "MongoDB URL (see https://godoc.org/labix.org/v2/mgo#Dial for format)")
+		cmd.Flags().String("mongo-database", "charts", "MongoDB database")
+		cmd.Flags().String("mongo-user", "", "MongoDB user")
+		cmd.Flags().Bool("debug", false, "verbose logging")
+	}
+}

cmd/chart-repo/delete.go

@@ -0,0 +1,30 @@
+/*
+Copyright (c) 2017-2018 Bitnami
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"github.com/spf13/cobra"
+)
+
+var deleteCmd = &cobra.Command{
+	Use: "delete [REPO NAME]",
+	Short: "delete a chart repository",
+	Run: func(cmd *cobra.Command, args []string) {
+		cmd.Help()
+		return
+	},
+}

cmd/chart-repo/sync.go

@@ -0,0 +1,69 @@
+/*
+Copyright (c) 2017-2018 Bitnami
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"os"
+
+	"github.com/spf13/cobra"
+	"github.com/sirupsen/logrus"
+	"github.com/kubeapps/common/datastore"
+)
+
+var syncCmd = &cobra.Command{
+	Use: "sync [REPO NAME] [REPO URL]",
+	Short: "add a new chart repository, and resync its charts periodically",
+	Run: func(cmd *cobra.Command, args []string) {
+		if len(args) != 2 {
+			logrus.Info("Need exactly two arguments: [REPO NAME] [REPO URL]")
+			cmd.Help()
+			return
+		}
+
+		mongoURL, err := cmd.Flags().GetString("mongo-url")
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		mongoDB, err := cmd.Flags().GetString("mongo-database")
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		mongoUser, err := cmd.Flags().GetString("mongo-user")
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		mongoPW := os.Getenv("MONGO_PASSWORD")
+		debug, err := cmd.Flags().GetBool("debug")
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		if debug {
+			logrus.SetLevel(logrus.DebugLevel)
+		}
+		mongoConfig := datastore.Config{URL: mongoURL, Database: mongoDB, Username: mongoUser, Password: mongoPW}
+		dbSession, err := datastore.NewSession(mongoConfig)
+		if err != nil {
+			logrus.Fatalf("Can't connect to mongoDB: %v", err)
+		}
+
+		if err = syncRepo(dbSession, args[0], args[1]); err != nil {
+			logrus.Fatalf("Can't add chart repository to database: %v", err)
+		}
+
+		logrus.Infof("Successfully added the chart repository %s to database", args[0])
+	},
+}

cmd/chart-repo-sync/sync.go → cmd/chart-repo/utils.go

@@ -1,5 +1,5 @@
 /*
-Copyright (c) 2017 Bitnami
+Copyright (c) 2017-2018 Bitnami
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -21,13 +21,11 @@ import (
 	"bytes"
 	"compress/gzip"
 	"errors"
-	"flag"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"net/http"
 	"net/url"
-	"os"
 	"path"
 	"strings"
 	"sync"
@@ -36,10 +34,10 @@ import (
 	"github.com/disintegration/imaging"
 	"github.com/ghodss/yaml"
 	"github.com/jinzhu/copier"
-	"github.com/kubeapps/common/datastore"
 	log "github.com/sirupsen/logrus"
 	"gopkg.in/mgo.v2/bson"
 	helmrepo "k8s.io/helm/pkg/repo"
+	"github.com/kubeapps/common/datastore"
 )
 
 const (
@@ -57,46 +55,10 @@ type httpClient interface {
 	Do(req *http.Request) (*http.Response, error)
 }
 
-var dbSession datastore.Session
-
 var netClient httpClient = &http.Client{
 	Timeout: time.Second * 10,
 }
 
-func main() {
-	flag.Usage = func() {
-		fmt.Fprintf(os.Stderr, "Usage: %s [options] [REPO NAME] [REPO URL]\n", os.Args[0])
-		flag.PrintDefaults()
-	}
-	dbURL := flag.String("mongo-url", "localhost", "MongoDB URL (see https://godoc.org/labix.org/v2/mgo#Dial for format)")
-	dbName := flag.String("mongo-database", "charts", "MongoDB database")
-	dbUsername := flag.String("mongo-user", "", "MongoDB user")
-	dbPassword := os.Getenv("MONGO_PASSWORD")
-	debug := flag.Bool("debug", false, "verbose logging")
-	flag.Parse()
-
-	if *debug {
-		log.SetLevel(log.DebugLevel)
-	}
-
-	if flag.NArg() != 2 {
-		flag.Usage()
-		os.Exit(2)
-	}
-
-	mongoConfig := datastore.Config{URL: *dbURL, Database: *dbName, Username: *dbUsername, Password: dbPassword}
-	var err error
-	dbSession, err = datastore.NewSession(mongoConfig)
-	if err != nil {
-		log.WithFields(log.Fields{"host": *dbURL}).Fatal(err)
-	}
-
-	if err := syncRepo(flag.Arg(0), flag.Arg(1)); err != nil {
-		log.WithError(err).Error("sync failed")
-		os.Exit(1)
-	}
-}
-
 // Syncing is performed in the following steps:
 // 1. Update database to match chart metadata from index
 // 2. Concurrently process icons for charts (concurrently)
@@ -106,7 +68,7 @@ func main() {
 // These steps are processed in this way to ensure relevant chart data is
 // imported into the database as fast as possible. E.g. we want all icons for
 // charts before fetching readmes for each chart and version pair.
-func syncRepo(repoName, repoURL string) error {
+func syncRepo(dbSession datastore.Session, repoName, repoURL string) error {
 	url, err := url.ParseRequestURI(repoURL)
 	if err != nil {
 		log.WithFields(log.Fields{"url": repoURL}).WithError(err).Error("failed to parse URL")
@@ -119,7 +81,7 @@ func syncRepo(repoName, repoURL string) error {
 	}
 
 	charts := chartsFromIndex(index, repo{Name: repoName, URL: repoURL})
-	err = importCharts(charts)
+	err = importCharts(dbSession, charts)
 	if err != nil {
 		return err
 	}
@@ -133,7 +95,7 @@ func syncRepo(repoName, repoURL string) error {
 	log.Debugf("starting %d workers", numWorkers)
 	for i := 0; i < numWorkers; i++ {
 		wg.Add(1)
-		go importWorker(&wg, iconJobs, chartFilesJobs)
+		go importWorker(dbSession, &wg, iconJobs, chartFilesJobs)
 	}
 
 	// Enqueue jobs to process chart icons
@@ -169,6 +131,15 @@ func syncRepo(repoName, repoURL string) error {
 	return nil
 }
 
+func deleteRepo(dbSession datastore.Session, repoName string) error {
+	err := removeCharts(dbSession, repoName)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func fetchRepoIndex(repoURL *url.URL) (*helmrepo.IndexFile, error) {
 	// use a copy of the URL struct so we don't modify the original
 	indexURL := *repoURL
@@ -231,7 +202,7 @@ func newChart(entry helmrepo.ChartVersions, r repo) chart {
 	return c
 }
 
-func importCharts(charts []chart) error {
+func importCharts(dbSession datastore.Session, charts []chart) error {
 	var pairs []interface{}
 	var chartIDs []string
 	for _, c := range charts {
@@ -259,23 +230,36 @@ func importCharts(charts []chart) error {
 	return err
 }
 
-func importWorker(wg *sync.WaitGroup, icons <-chan chart, chartFiles <-chan importChartFilesJob) {
+func removeCharts(dbSession datastore.Session, repoName string) error {
+	db, closer := dbSession.DB()
+	defer closer()
+	bulk := db.C(chartCollection).Bulk()
+
+	bulk.RemoveAll(bson.M{
+		"repo.name": repoName,
+	})
+
+	_, err := bulk.Run()
+	return err
+}
+
+func importWorker(dbSession datastore.Session, wg *sync.WaitGroup, icons <-chan chart, chartFiles <-chan importChartFilesJob) {
 	defer wg.Done()
 	for c := range icons {
 		log.WithFields(log.Fields{"name": c.Name}).Debug("importing icon")
-		if err := fetchAndImportIcon(c); err != nil {
+		if err := fetchAndImportIcon(dbSession, c); err != nil {
 			log.WithFields(log.Fields{"name": c.Name}).WithError(err).Error("failed to import icon")
 		}
 	}
 	for j := range chartFiles {
 		log.WithFields(log.Fields{"name": j.Name, "version": j.ChartVersion.Version}).Debug("importing readme and values")
-		if err := fetchAndImportFiles(j.Name, j.Repo, j.ChartVersion); err != nil {
+		if err := fetchAndImportFiles(dbSession, j.Name, j.Repo, j.ChartVersion); err != nil {
 			log.WithFields(log.Fields{"name": j.Name, "version": j.ChartVersion.Version}).WithError(err).Error("failed to import files")
 		}
 	}
 }
 
-func fetchAndImportIcon(c chart) error {
+func fetchAndImportIcon(dbSession datastore.Session, c chart) error {
 	if c.Icon == "" {
 		log.WithFields(log.Fields{"name": c.Name}).Info("icon not found")
 		return nil
@@ -313,7 +297,7 @@ func fetchAndImportIcon(c chart) error {
 	return db.C(chartCollection).UpdateId(c.ID, bson.M{"$set": bson.M{"raw_icon": b.Bytes()}})
 }
 
-func fetchAndImportFiles(name string, r repo, cv chartVersion) error {
+func fetchAndImportFiles(dbSession datastore.Session, name string, r repo, cv chartVersion) error {
 	chartFilesID := fmt.Sprintf("%s/%s-%s", r.Name, name, cv.Version)
 	db, closer := dbSession.DB()
 	defer closer()