From e4e7f2f6d0f16ece2d326b6e25318e2b55a47d5c Mon Sep 17 00:00:00 2001 From: Ceyhun Date: Sun, 19 Jun 2022 23:24:23 +0200 Subject: [PATCH] Improve rucio-dataset-mon-go html (#150) --- src/go/rucio-dataset-mon-go/configs/setup.go | 38 ---- .../controllers/dataset_controller.go | 163 ++++++---------- src/go/rucio-dataset-mon-go/main.go | 4 +- .../models/dataset_model.go | 20 +- src/go/rucio-dataset-mon-go/mongo/mongo.go | 25 +++ src/go/rucio-dataset-mon-go/mongo/setup.go | 36 ++++ .../responses/error_dt_response.go | 8 - .../routes/main_router.go | 6 +- .../static/img/oc_icon.png | Bin 0 -> 11588 bytes src/go/rucio-dataset-mon-go/static/index.html | 177 ++++++++++++++---- src/go/rucio-dataset-mon-go/utils/utils.go | 54 ++++++ 11 files changed, 331 insertions(+), 200 deletions(-) delete mode 100644 src/go/rucio-dataset-mon-go/configs/setup.go create mode 100644 src/go/rucio-dataset-mon-go/mongo/mongo.go create mode 100644 src/go/rucio-dataset-mon-go/mongo/setup.go delete mode 100644 src/go/rucio-dataset-mon-go/responses/error_dt_response.go create mode 100644 src/go/rucio-dataset-mon-go/static/img/oc_icon.png create mode 100644 src/go/rucio-dataset-mon-go/utils/utils.go diff --git a/src/go/rucio-dataset-mon-go/configs/setup.go b/src/go/rucio-dataset-mon-go/configs/setup.go deleted file mode 100644 index db18378b..00000000 --- a/src/go/rucio-dataset-mon-go/configs/setup.go +++ /dev/null @@ -1,38 +0,0 @@ -package configs - -import ( - "context" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" - "log" - "time" -) - -// ConnectDB mongo.Client which is MongoDB connection client -func ConnectDB() *mongo.Client { - client, err := mongo.NewClient(options.Client().ApplyURI(EnvMongoURI())) - if err != nil { - log.Fatal(err) - } - - ctx, _ := context.WithTimeout(context.Background(), time.Duration(EnvConnTimeout())*time.Second) - if err = client.Connect(ctx); err != nil { - log.Fatal(err) - } - - //ping the database - if err := client.Ping(ctx, nil); err != nil { - log.Fatal(err) - } - log.Println("Connected to MongoDB") - return client -} - -// DB MongoDb Client instance(mongo.Client) -var DB = ConnectDB() - -// GetCollection returns Mongo db collection with the given collection name -func GetCollection(client *mongo.Client, collectionName string) *mongo.Collection { - collection := client.Database(EnvMongoDB()).Collection(collectionName) - return collection -} diff --git a/src/go/rucio-dataset-mon-go/controllers/dataset_controller.go b/src/go/rucio-dataset-mon-go/controllers/dataset_controller.go index 730a1ec2..83c5cdf8 100644 --- a/src/go/rucio-dataset-mon-go/controllers/dataset_controller.go +++ b/src/go/rucio-dataset-mon-go/controllers/dataset_controller.go @@ -5,158 +5,109 @@ import ( "encoding/json" "github.com/dmwm/CMSMonitoring/src/go/rucio-dataset-mon-go/configs" "github.com/dmwm/CMSMonitoring/src/go/rucio-dataset-mon-go/models" - "github.com/dmwm/CMSMonitoring/src/go/rucio-dataset-mon-go/responses" + "github.com/dmwm/CMSMonitoring/src/go/rucio-dataset-mon-go/mongo" + "github.com/dmwm/CMSMonitoring/src/go/rucio-dataset-mon-go/utils" "github.com/gin-gonic/gin" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" "io" "log" "net/http" - "strings" "time" ) var ( - // connection instance for the "datasets" collection - datasetCollection = configs.GetCollection(configs.DB, configs.GetEnvVar("COLLECTION_DATASETS")) + // connection instance for the "datasets" datasetsDb + datasetsDb = mongo.GetCollection(mongo.DBClient, configs.GetEnvVar("COLLECTION_DATASETS")) GlobalTotalRecCount int64 ) -// MiddlewareReqHandler handles CORS and HTTP request settings for the context router -func MiddlewareReqHandler() gin.HandlerFunc { - return func(c *gin.Context) { - //c.Writer.Header().Set("Content-Type", "application/json") - c.Writer.Header().Set("Access-Control-Allow-Origin", "*") - c.Writer.Header().Set("Access-Control-Allow-Credentials", "true") - c.Writer.Header().Set("Access-Control-Allow-Headers", "Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization, accept, origin, Cache-Control, X-Requested-With") - c.Writer.Header().Set("Access-Control-Allow-Methods", "POST, OPTIONS, GET, PUT, DELETE") - if c.Request.Method == "OPTIONS" { - c.AbortWithStatus(204) - return - } - c.Next() - } -} - -// errorResponse returns error response with given msg and error -func errorResponse(c *gin.Context, msg string, err error) { - log.Printf("[ERROR] %s %s", msg, err) - c.JSON(http.StatusInternalServerError, - responses.ErrorResponseStruct{ - Status: http.StatusInternalServerError, - Message: msg, - Data: map[string]string{"data": err.Error()}, - }) -} - -// getRequestBody parse datatable request +// getRequestBody parse datatables request func getRequestBody(c *gin.Context) models.DataTableRequest { // === ~~~~~~ Decode incoming DataTable request json ~~~~~~ === log.SetFlags(log.LstdFlags | log.Lshortfile) body, err := io.ReadAll(c.Request.Body) if err != nil { - errorResponse(c, "Request body read failed", err) + utils.ErrorResponse(c, "Request body read failed", err) } log.Printf("Request Body:\n%#v\n", string(body)) var dataTableRequest models.DataTableRequest if err := json.Unmarshal(body, &dataTableRequest); err != nil { - errorResponse(c, "Unmarshal request body failed", err) + utils.ErrorResponse(c, "Unmarshal request body failed", err) } return dataTableRequest } -// getTotalRecCount total document count in the collection -func getTotalRecCount(ctx context.Context, c *gin.Context) int64 { - if GlobalTotalRecCount == 0 { - countTotal, err := datasetCollection.CountDocuments(ctx, bson.M{}) - if err != nil { - errorResponse(c, "TotalRecCount query failed", err) - } - GlobalTotalRecCount = countTotal - } - log.Printf("[INFO] Total Count %d", GlobalTotalRecCount) - return GlobalTotalRecCount -} - -// getFilteredRecCount filtered document count in the collection -func getFilteredRecCount(ctx context.Context, c *gin.Context, findQuery bson.M) int64 { - filteredRecCount, err := datasetCollection.CountDocuments(ctx, findQuery) - if err != nil { - errorResponse(c, "FilteredRecCount query failed", err) - } - return filteredRecCount -} - -// convertOrderEnumToMongoInt converts DataTable enums ("asc" and "desc") to Mongo sorting integer definitions (1,-1) -func convertOrderEnumToMongoInt(dir string) int { - switch strings.ToLower(dir) { - case "asc": - return 1 - case "desc": - return -1 - default: - return 0 - } -} - -// generateSortOrder creates sort order for the Mongo query by iterating over DataTable json request -func generateSortOrder(dataTableRequest models.DataTableRequest) *options.FindOptions { +// getSortBson creates sort object which support multiple column sort +func getSortBson(dataTableRequest models.DataTableRequest) bson.D { orders := dataTableRequest.Orders columns := dataTableRequest.Columns - sortOpts := options.Find() - sortOpts.Limit = &dataTableRequest.Length - sortOpts.Skip = &dataTableRequest.Start + sortBson := bson.D{} for _, order := range orders { - intSortDirection := convertOrderEnumToMongoInt(order.Dir) + intSortDirection := utils.ConvertOrderEnumToMongoInt(order.Dir) if intSortDirection != 0 { - sortOpts = sortOpts.SetSort( - bson.D{{ - Key: columns[order.Column].Data, // column name - Value: intSortDirection, // column direction as int value - }}, - ) + sortBson = append(sortBson, bson.E{ + Key: columns[order.Column].Data, // column name + Value: intSortDirection, // column direction as int value (0/1) + }) } } - return sortOpts + // Always add unique id column at the end to be able fetch non-unique columns in order + sortBson = append(sortBson, bson.E{Key: "_id", Value: 1}) + return sortBson } -// generateFindQuery creates main search query using regex by default -func generateFindQuery(dataTableRequest models.DataTableRequest) bson.M { +// getSearchBson creates main search query using regex by default +func getSearchBson(dataTableRequest models.DataTableRequest) bson.M { // DataTable main search request struct dtMainSearch := dataTableRequest.Search if dtMainSearch.Value == "" { return bson.M{} } else { log.Printf("[INFO] dtMainSearch.Value is : %s", dtMainSearch.Value) - var findQuery []bson.M - findQuery = append(findQuery, bson.M{"dataset": primitive.Regex{Pattern: dtMainSearch.Value, Options: "im"}}) - // i: case insensitive, m: can use ^ and $. Ref: // https://www.mongodb.com/docs/v5.0/reference/operator/query/regex/ // TODO add individual column search - return bson.M{"$and": findQuery} + return bson.M{"$and": bson.M{"Dataset": primitive.Regex{Pattern: dtMainSearch.Value, Options: "im"}}} } } -// paginateResults get query results efficiently -func paginateResults(ctx context.Context, c *gin.Context, findQuery bson.M, sortOptsOfFind *options.FindOptions) []models.Dataset { - var datasets []models.Dataset - datasetResults, err := datasetCollection.Find(ctx, findQuery, sortOptsOfFind) +// getTotalRecCount total document count in the datasetsDb +func getTotalRecCount(ctx context.Context, c *gin.Context) int64 { + if GlobalTotalRecCount == 0 { + countTotal, err := mongo.GetCount(ctx, datasetsDb, bson.M{}) + if err != nil { + utils.ErrorResponse(c, "TotalRecCount query failed", err) + } + GlobalTotalRecCount = countTotal + } + log.Printf("[INFO] Total Count %d", GlobalTotalRecCount) + return GlobalTotalRecCount +} + +// getFilteredRecCount filtered document count in the datasetsDb +func getFilteredRecCount(ctx context.Context, c *gin.Context, findQuery bson.M) int64 { + filteredRecCount, err := mongo.GetCount(ctx, datasetsDb, findQuery) if err != nil { - errorResponse(c, "datasetCollection.Find query failed", err) + utils.ErrorResponse(c, "FilteredRecCount query failed", err) } + return filteredRecCount +} - // reading from the db in an optimal way - defer func(results *mongo.Cursor, ctx context.Context) { - if err := results.Close(ctx); err != nil { - errorResponse(c, "MongoDB cursor failed", err) - } - }(datasetResults, ctx) +// getQueryResults get query results efficiently +func getQueryResults(ctx context.Context, c *gin.Context, dataTableRequest models.DataTableRequest) []models.Dataset { + var datasets []models.Dataset + searchQuery := getSearchBson(dataTableRequest) + sortQuery := getSortBson(dataTableRequest) + limit := dataTableRequest.Length + skip := dataTableRequest.Start - for datasetResults.Next(ctx) { + datasetsCursor, err := mongo.GetResults(ctx, datasetsDb, searchQuery, sortQuery, skip, limit) + if err != nil { + utils.ErrorResponse(c, "datasetsDb.Find query failed", err) + } + for datasetsCursor.Next(ctx) { var singleDataset models.Dataset - if err = datasetResults.Decode(&singleDataset); err != nil { - errorResponse(c, "datasetResults.Decode failed", err) + if err = datasetsCursor.Decode(&singleDataset); err != nil { + utils.ErrorResponse(c, "datasetResults.Decode failed", err) } datasets = append(datasets, singleDataset) } @@ -171,10 +122,8 @@ func GetDatasets() gin.HandlerFunc { dataTableRequest := getRequestBody(c) totalRecCount := getTotalRecCount(ctx, c) - sortOptsOfFind := generateSortOrder(dataTableRequest) - findQuery := generateFindQuery(dataTableRequest) - filteredRecCount := getFilteredRecCount(ctx, c, findQuery) - datasets := paginateResults(ctx, c, findQuery, sortOptsOfFind) + filteredRecCount := getFilteredRecCount(ctx, c, getSearchBson(dataTableRequest)) + datasets := getQueryResults(ctx, c, dataTableRequest) // Send response in DataTable required format // - Need to return exactly same "Draw" value that DataTable sent in incoming request diff --git a/src/go/rucio-dataset-mon-go/main.go b/src/go/rucio-dataset-mon-go/main.go index 2a5adc75..c4aae4f6 100644 --- a/src/go/rucio-dataset-mon-go/main.go +++ b/src/go/rucio-dataset-mon-go/main.go @@ -3,8 +3,8 @@ package main import ( "flag" "fmt" - "github.com/dmwm/CMSMonitoring/src/go/rucio-dataset-mon-go/configs" "github.com/dmwm/CMSMonitoring/src/go/rucio-dataset-mon-go/controllers" + "github.com/dmwm/CMSMonitoring/src/go/rucio-dataset-mon-go/mongo" "github.com/dmwm/CMSMonitoring/src/go/rucio-dataset-mon-go/routes" "golang.org/x/sync/errgroup" "log" @@ -40,7 +40,7 @@ func main() { } // connect to database - configs.ConnectDB() + mongo.GetMongoClient() controllers.GitVersion = gitVersion controllers.ServerInfo = info() diff --git a/src/go/rucio-dataset-mon-go/models/dataset_model.go b/src/go/rucio-dataset-mon-go/models/dataset_model.go index df9fee58..481d6f60 100644 --- a/src/go/rucio-dataset-mon-go/models/dataset_model.go +++ b/src/go/rucio-dataset-mon-go/models/dataset_model.go @@ -1,16 +1,14 @@ package models -import "go.mongodb.org/mongo-driver/bson/primitive" - // Dataset struct which includes Rucio and DBS calculated values type Dataset struct { - Id primitive.ObjectID `bson:"_id"` - RseType string `bson:"RSE_TYPE,omitempty" validate:"required"` - Dataset string `bson:"dataset,omitempty" validate:"required"` - LastAccess string `bson:"last_access"` - LastAccessMs int64 `bson:"last_access_ms"` - MaxDatasetSizeInRsesGB float64 `bson:"max_dataset_size_in_rses(GB)"` - MinDatasetSizeInRsesGB float64 `bson:"min_dataset_size_in_rses(GB)"` - SumDatasetSizeInRsesGB float64 `bson:"sum_dataset_size_in_rses(GB)"` - RSEs string `bson:"RSE(s)"` + RseType string `bson:"RseType,omitempty" validate:"required"` + Dataset string `bson:"Dataset,omitempty" validate:"required"` + LastAccess string `bson:"LastAccess"` + LastAccessMs int64 `bson:"LastAccessMs"` + MaxDatasetSizeInRsesGB float64 `bson:"MaxDatasetSizeInRsesGB"` + MinDatasetSizeInRsesGB float64 `bson:"MinDatasetSizeInRsesGB"` + AvgDatasetSizeInRsesGB float64 `bson:"AvgDatasetSizeInRsesGB"` + SumDatasetSizeInRsesGB float64 `bson:"SumDatasetSizeInRsesGB"` + RSEs string `bson:"RSEs"` } diff --git a/src/go/rucio-dataset-mon-go/mongo/mongo.go b/src/go/rucio-dataset-mon-go/mongo/mongo.go new file mode 100644 index 00000000..99de6cb4 --- /dev/null +++ b/src/go/rucio-dataset-mon-go/mongo/mongo.go @@ -0,0 +1,25 @@ +package mongo + +import ( + "context" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" +) + +// GetResults returns cursor of aggregate query results +func GetResults(ctx context.Context, collection *mongo.Collection, match bson.M, sort bson.D, skip int64, limit int64) (*mongo.Cursor, error) { + pipeline := []bson.M{ + {"$match": match}, + {"$sort": sort}, + {"$skip": skip}, + {"$limit": limit}, + } + cursor, err := collection.Aggregate(ctx, pipeline) + return cursor, err +} + +// GetCount returns count of query result +func GetCount(ctx context.Context, collection *mongo.Collection, match bson.M) (int64, error) { + count, err := collection.CountDocuments(ctx, match) + return count, err +} diff --git a/src/go/rucio-dataset-mon-go/mongo/setup.go b/src/go/rucio-dataset-mon-go/mongo/setup.go new file mode 100644 index 00000000..9067a609 --- /dev/null +++ b/src/go/rucio-dataset-mon-go/mongo/setup.go @@ -0,0 +1,36 @@ +package mongo + +import ( + "context" + "github.com/dmwm/CMSMonitoring/src/go/rucio-dataset-mon-go/configs" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "log" + "time" +) + +// DBClient MongoDb Client instance(mongo.Client) +var DBClient = GetMongoClient() + +// GetMongoClient returns mongo client +func GetMongoClient() *mongo.Client { + var err error + var client *mongo.Client + opts := options.Client() + opts.ApplyURI(configs.EnvMongoURI()) + opts.SetMaxPoolSize(100) + opts.SetConnectTimeout(time.Duration(configs.EnvConnTimeout()) * time.Second) + if client, err = mongo.Connect(context.Background(), opts); err != nil { + log.Fatal(err) + } + if err := client.Ping(context.Background(), nil); err != nil { + log.Fatal(err) + } + return client +} + +// GetCollection returns Mongo db collection with the given collection name +func GetCollection(client *mongo.Client, collectionName string) *mongo.Collection { + collection := client.Database(configs.EnvMongoDB()).Collection(collectionName) + return collection +} diff --git a/src/go/rucio-dataset-mon-go/responses/error_dt_response.go b/src/go/rucio-dataset-mon-go/responses/error_dt_response.go deleted file mode 100644 index fbdaa237..00000000 --- a/src/go/rucio-dataset-mon-go/responses/error_dt_response.go +++ /dev/null @@ -1,8 +0,0 @@ -package responses - -// ErrorResponseStruct custom response struct, used in case of error -type ErrorResponseStruct struct { - Status int `json:"status"` - Message string `json:"message"` - Data map[string]string `json:"data"` -} diff --git a/src/go/rucio-dataset-mon-go/routes/main_router.go b/src/go/rucio-dataset-mon-go/routes/main_router.go index fb235686..6fbb14c8 100644 --- a/src/go/rucio-dataset-mon-go/routes/main_router.go +++ b/src/go/rucio-dataset-mon-go/routes/main_router.go @@ -2,6 +2,7 @@ package routes import ( "github.com/dmwm/CMSMonitoring/src/go/rucio-dataset-mon-go/controllers" + "github.com/dmwm/CMSMonitoring/src/go/rucio-dataset-mon-go/utils" "github.com/gin-gonic/gin" "net/http" ) @@ -12,11 +13,12 @@ func MainRouter() http.Handler { e.Use(gin.Recovery()) // REST - e.Use(controllers.MiddlewareReqHandler()) + e.Use(utils.MiddlewareReqHandler()) e.POST("/api/datasets", controllers.GetDatasets()) + e.Static("/static/img", "./static/img") // Static - e.LoadHTMLGlob("static/*") + e.LoadHTMLGlob("static/index.html") e.GET("/", controllers.GetIndexPage) e.GET("/serverinfo", controllers.GetServerInfo) diff --git a/src/go/rucio-dataset-mon-go/static/img/oc_icon.png b/src/go/rucio-dataset-mon-go/static/img/oc_icon.png new file mode 100644 index 0000000000000000000000000000000000000000..5ca9c1271a5082d0187d85ed2b12a9d2383e2bd9 GIT binary patch literal 11588 zcmV-KExXc*P)t=B0030=|Nr~{`Tzh-002<{0DA-rb^rhN00?^k4s!zwcmM!S002Y)07&L45dZ)@ zIXi*&`Ty)H0ORKF!pPeC4$#y*wfF_P>Jr9iRLyTv%AsmtDfv^ zV*c^)*xBfAa-`!rCRJIC=YDea+0ErbGxov6Jw|;QB5N2Nao$@|-c>xPt-htH$|5Iw zoSd+Vi?%d3f<{Y^IzNfxH!Wglo6XhPGdFE;bCuiOpY03ZNKL_t(|ob7#$LmJ1{ z_w3Hj>@s|~fNUZNX}94^77!Faa3KgN1d)6gW70Hw<87_APi^m4?M>Re?S0-q{5`Xv z0hO_&vSFH$}V$u&Ybi6ojG$Bp=-I8Yq^$dxt43WmTS3|Yq^$dxt43WmTUR{ zM+78FnnGmZLl5LV3uA~WD1MK9=z+X{VU%oEQc3)w$MJqe#7rXb)~nP#a`rnx5RoJ> zQT&u|elqkj5l??}`X`BgN|Q_#UwSK|&l1wsiXypQvLqLkqz^o}cPESyF}w5tBggPeL$O6xxiKFVkngq720z77@zY@X7)bq)#L0FvS;` zHO;M>?`$=SGFp5ZpaJ2i9(90Q`cxDR%5Y1JKs2HW6hSwDoV(CdPc-Clcd;4)I zDx+~`R1&Emb=^v!2s7HbgTU7SgMa#aad3N**!8o(pLo}gjNrF1FhvkCTGRDL28q4b zGogM<-PzeW%A&D?ixQAgF;-exn)`8XY2|SiJhvpBY7Q!fpTNKeD5?bZqP+u=0lL&5 zKV6!eTYQowsOK?`EJWdYn$I)j+#kdMnqPQ}8$v*O`oQaPxq@Fl+nMlAIGwleWHIW@ z5onnIq9_egl%m_Q?w_%S!lkdhUguX|Ic)00(R^4?-eOc#(bf#(dB%7CVo^i$bK~8L zAxdfWOOM00Tg|tBm(MLsxK&4R<;-+Z)-_MXev`s~AG76zGjuH~5~|u%$GmmJ{G4;j z;mnR}x}u0h4X?dQRWd|8?}wMbp|R^MG|tJsl~H3ib|?|UQ+t-A%H14)v@%(Dai0()Cb`} z`vM@$Ip>$-DQR8i(-J25A)<)l`9D$=&@c-yEx_0aER7R7wCk9*gSm^6aC$!xNG?Tn zZ|?MC0$Nn3UEzMjk4fPpyLA$9J8hXzM0MWcaJZ-nOTesv!8Q4}WJ;kPfN)77G&MLYc1 zeb3l8bUb@ zh5I}^spts#LqX+KQ4t+Y*;j;fadj+3!ER;Hll!4d|1Kc8XnDG9aTG-*A!hT3T=jr> zUIkDTv(s<_&dE6)3{K6XJ|JX~D{3pADk|4qL3^&}6QUrE9$rKqKm)TWvz77nF$6K1 zU)e3@^ZA?AQEn)ahwh+#iEA=Q1XOgYsz2ljU-Chi`Y3oazuy}@8&pj7tD_8{@3_Dl z{-7Hi5B2-Z_7^Vqty?!Iz4=irS137xAy;$|65t}CJ@3+-+pn%5<$|U{0NqtrcGb{C z?94iIj*z4D_M8Ijmh)`DxBWkYqNNwbd^Dqa4t`#WlY)}&Yrhl|A#mcEQ)o=_0RH%xG# z=4zNWYDL$*NsM_^(R3xl#T)sdpH%GdNADYGqm9V4R`H+Q09W%w^b!ncZS?Q zqRt}|(S|$ZcW?9yM}voj;{`)sui=5i5|AJRy%Nx~>QFk0rKr>IcV7ex7rz@VPY@*4 zUDJM7RO${z5=Zk3pjqAE6b#v41EHZ3#UtnF7*-hQX{OfGXmD1Vp*F)P zKCGf)d0mWR4!_?%Km7XzqQ2$~aLyfq=OVvx5fotevW@ezuq)&al8&MQ1uZ#v;B{{7 z@n9{Fn(JT+A)6K5Y-+=k?=<(hmlMn4(5J{y$?gxS+arvkII(JRdQZsjtjlj~iYY?Q zxoVljjyml;HC3iaWX>6y_6OazjVc-X$lu(Luw)~3@PpQmKAZ_fQoj0a_1jocpu<44 zfbKw;cT|U?Z6Rx00$eNs;tfJH>f%f?d+4fLQ@m;qIish4pb&1KI@KWHLbu)Vlw&n% zsLKQTiP2%jc$HuZ8|q*L&Lwl(dA@pvtUY?F=&%tWx;?RiG(|vpaJ(JG+JBHr&fT^% z*mZ|&T{#9(L-hqaSPZD@@XqBWMH%)yF(x?i>Oeo%Mgofz-*-No`fQQmVT!VmXB4g2 za*`&(4PUr}Q>$mnh;mqU<^$FYFZQB70C5z>R;^i`_N&eu8op4KKpAM+A?}0YdxVNY z?kEbF9^&XQ5k{U-psX>OQ&c(cnF<1SXip-Ln|XB+DHiyDqJm%R%A{MqpXOH_>J;cY z<+^(zNzglRSZaaaQbuDF5ov^NKu(cY_B*kvoWV_^U=&osAo{u4z@6Z4##bMhK)ESUzIXIeH1hzvHq&a=Xskronc!e&57I2%ow> zOUclfcqSq&3jn*({BRcyi|&~_6mo@!dZV19V;!81z`}NW!jd2avd)L(Y>s#~#g@P1 zVs6)zFA`g{AF3v7&5W1{$u~L)prRBFdT)6rzZf>3+?SvdOpjx4=NR^R@E+s$AWIZt$xYHpeU2@k?IGB*5{k#SyQhhu_*N?+7|{79)CF>&F7=TXO!pp zY_N9b2k)aY1{2@yGFJc|ak>_pc) zkUGBia@YD_($E0S)!T3frysDv8^#O;?N8qPD1`y9t4j!d>BzltE)sfXpH};Dcu;gj z^#ek@@HHaUaHe2Jeu?{r!rSwsr{9Su)b3PWuFtchY>z@DvGO_<#?+ACRc~X0RdvB9i<(6MDc?18ft&6L zcuY}9@}Q!F8ya0jOB9W9&BjlqDk!>Oo3bs2k$mbz`HhN;9`zeZ25J_z*@L~Vp9+hr z=X|cF4#3$Fv&Tjt2-ZG7mC)XOkMn+g znskS?30FI*iqQl{NL1YP_P~K(#`rIXQadN3A8;BUqyFwv*iDx{i_RKY< zI-hp#SKn}W78&h!(Ta#`zkwS{VlrOz1Udd~23upy-`b5;H$34nuBFJATXi`-%S)wd zb}yTI`e4$#vy!(8YD6jKaUAtJD{$y9&cRwn@#hYk>jhF^1u#@ycB-m#Vd=?}7t52L zxo+^scKN5GOrU~PtgfsE9Zr|W<9xPQ%`-vP z7`G9iL)mlb3K7gezCbjx^?G(Mz+o0ogvFL_JKQdZEjaf$+LIXrMTvI&%Fy>`#-2`* zB=tyx?+pWvnqT5|39Gp=6qR6Qcj|z`#V|JvDL}C(TFHKi7aaB03CO@KG3z0h6nH0v znWk3p6>7&(T(s;;lDn^F45w#P#z9%b%9vXgB04dQMv_>DMzfREVlfAiZ~To9o|L#) z0MhqcN4n#=z@bq};#-E^NTRXpjzqHA(2ZllQ85QnD>pqiJ=Kr2CRrgQ`P_&UGFTiB z^KnO=sTh$p)GprOxwqn8T^s*-{W*fuWnGWVrU@Ifcmpm?Rt#f}C<<1%sl;BuKle*4 zFm8N$?}CwtaqT<9tgNx|Dj?xX2Kb_3rg#$H`jr&B@eDJvIJs+;3Vl>U89~YCCYUW5 zmwdvc;Wb^?*WSSYokfI!*=4NgAQE#;b$m>7@|E$RuDt23GsvThRA?jruKw z`f##knvcc@L)If&H_S$coMx3Mc-+-ptDZ7@BZ}Ut68L=yOV8@MIa|ZpxK~oem-)#6 zI){`~gXepO^&6|PdTiy!U4(}e5B*uv-h;|)MIx1BI>4n?a6mH)T@a` zV2Pzny`09{SV4vBMSMSh_ewVoFP3gFi#?AFSl_z{A~!u#Zvpj0<2`-wseDhyGz)Qr z&u>rhyd`KUv5({o6L?JULZ-f7L!5JrF({F0*0s9XCd}wpa)DLFUK0Mwqh$7;^@pD~2D5nQAId};5-Pb!y7Ym60G=VS=(`o8I zZ34j_mthlab4xeO^$hk6)_@;P0@sW_(1eNt6A3zLMNA_RAfm)ZJRt{T77&&go|&zf zX6ruzA`A&fjB2}{35V-O7JUF1D9DttS~F5HwlbH6`Tjx$1`e#Y)-odpKhy{r=D|b_ z<(xTQXH@(Uii!$m#})I{L4u7*$SA{<9X>L3{coExR(Of!;BS)A3P1G+)bt&yX>kNW zlIR&9yySc@<(o%%bKNxbb$+nb>h&>^eQ)gp z(09nY`F09WU`1}L$M<@H51-pgWC4++f3Fy+=Z68(UYepftU&_&Rnqu|rXaEK086!3Vsx(N66-t>Z~>a)dIc%*ZQj4G%5SVPOwM-xgh z4WR6t&Efod>2Zu7s;SPeI-H&dr6N24f%8HRXwr%2hS^Bec$kOfEXVHt<-uPzO57)k zeP#&Mtk1ce-sPpo|M~~qH#g@{9ZtuOReX_powp59W^_XhI_%!qyGnNo0aSlxn{v(7 zyV4oq+>S%F&0|5-U}&ocmHwk z@2%OmkAYypoX z09K!O+Tq%ZGs@0q8KgT;(LxM;nSB>|U5WytZ&eLVEOvWw(W>1?(Ia)rl|wSF%{Fws zv6e<{8)Yc#RCAU?l~L5;$_6e(e+&>RnOC#v3KEh!JJ^l^J7{U z8>PS`{6tIqXf>F(%FL~$Zno(N%k@pHX`F#cxrMF~A zx#)F8Ce2jL0_#qC!itHN(Bjb|5=O?;gXE4HoO&(F^!!l-07pa+auY$j{hzyj zSwc&y`c$?G1z^R_tIy6lk$}Z3y!tqv`1ic(w?8B`gyr)+;Dqzl0Q{M<*;lC((p@n~ zdBsS(vg#cG1}Qrkn!dEmReZ=iVs~OIu^p_X(5l$;Z##ii|@anh_ZzOTC`V;sUiSHl$!{MKHR{2oQ{({kM zyZy8VAE6^h^)xPM1ZkN>OE)9;R;Pme)WV3XOAGvB8kq~p`gUgbE%Z8nA5n;TK#v%` z0L=jhA-C+Uj+Vxizy*cw3rqomKUMHfoVh`G? z{6H6YnfvgZ6bTbV(#d8+H_S*O5$6X{Luz=m>0pX4Q_%~x^O+SY7oXnWdyx-ZQBCe| z!mf~C-9;B}1>rSON$IVmLi5g$byVxk-UtoUJ^WY~<#;{kErHMU;aal!Tu(*JKV~;6 zL17oT^4$@8zhQBvpjS4}lm0l{& zM5*))E$pG?acTiC7a1+u{eEY*x0?o~K6VWFy>O3dG7ErpheEb!&(zSm%kQ610z&N8 zV-#Q8Z@dCEEBnc~#?22DUF1g=Myfx>OvDLYUzabqXDZEG5O5=h#;wdcFB12;eowI% zd0;ZLZ|XWAAzu3jK-dw2K+yv?Mc%T(zKd2|vExD|646tC?61YSo8k}%!uzTleB3CV zA^`i6W8G}cTugoPCK&aFSoLP#^HC#LPy_+wd7M@2-am z(9b&Uw6pZjf@wzdRx=rw2-vJ-9rP3=a^wz9dqzrZOE_6cH8K~I2!LVG{7sj~J3r)j z|0s&4Rlk4gFK4&ros>wrY?!I(+_hNMVYrsrw)=y&`Y(}6p`1+n zu!T|i0Y!dRRU!_*-@QzSoGapL@^W;;NRmrdvm{0L^08`ktfqjM3oU{U?lVQL-U0EK z0N@i6Vx2GUC~6(e1muGePH!f*zWZm+7V^9OnSaM6u=NgXf?zCZ00Sj{U;9^RnAc@I zIa4vsLyRtVtpPK;bHvphyHx$62u1K2?u9o_P0;kE>Ulzp_wy<>seVNigrnR9$CsdV?|&1U1aK` z5Jq7}t;GbqN|eh6_d>WE90*y5KWM+p_k1(umlXg6FMqx7S;SJz3E*cDDX^BTvl0M5 z=3IC9HFD7bvG+C|IHN>lB009oRxiz&G9=`@-pS8y{xx=G;}wcr(FM!tAjOAf*7aA7 zVs9S5*(NaVjlxRQ2s?Ul@^Ub>~FgdN^UQYxPSsc#SGP zF#&M#xL+$ScwY6rXg#_Yeybik-NLYAI_Q3lr8e{3Y(-K9;GLv9Z1FyaA#g?z(+%@r ztGpQ~Ih{ORMFVI7e5*6xLBVj*hQm541GrZW`Ta)&6%`ptH+8d-?7O=PB#_lXs`)D` ziqMyCf5;K03{VLLGctZlBdnb<872TvL?v*|>c9RlD_e<-g3)7_t;9DqB?Xo^Jf|Jf zt`=iL%yqld)xe6+aNoXZd;-xZzGbFbn^<9&e+Ire>hd~nPTJ~MR#ZZF0U5cXKr*mg z29S`@dd3Ez#i#sy7Zi2=1GEI#!=$=*>Pw4I@SItUL*+R@Sd<@8JaTog5JOAu5I+dr zS47@2MK`x*`hNrzeX>yW?tS|jUzvDIG{+AoRUvA_@-&}sBGQ}l@XaJz|9-|B%P15B z`G$wTJmshc+C66l1wB=Rp^47Xt|iYrJx#zul;vmm`U?Q=*0zKN-nvrRkOyb-1zCicEN z7;?rWK;~I5#G2Doh}B1F>9XM|oCBg{fwvN2i1jZ86$L4GA1yASjqNKd%8%c<{q~3G zq@t(xXSyJzMkOt?ZyJWaRfSODs9Ie;B}rJYL+pI>N&qSBngX6u7Z5pecAsTiaEI() zUuJO{EV!B3R~7T+kP1nPBUF%-Y(;c)O~KMYMlDcD zAbTjb6uWwgL8Kz)9VwP(Q)>{vEd_X9gR7Y`jY@Nq;35z*bskx`Uyx$9>25`lLWP_Y ztp9e^Q|c~wq$CK;d+B^)k>|NK!5ir78AOKlo}j37JD2C4WN4$LQ6$GZUjCXq&DZd< zZ=6{N09u*lYnGo;4;^(-m?#6Gw)gf+;oQ?7va5?dFi1f;75GcOYj+4%v?7vt(*SCe z7y-VDRLIZo+*mF}=}4KU!9sm>6(H*d3MkAfy8yqR*^i{m=Rd>+Lb27G%N6UeVuoUl zX}`nQO_Id`+Hm`W>I&-an8lk_*m>v0&T3%%OaSrC%vKIeo!469p@oB#-x`JI)z7l; zAg_zl-(|jbhX4YwPJKzdd7!_ll;c3W0E}wO)^C7uf1O_Mz5wQ_&g@2Uz)ADa|$L+ z2mlHq=>*?qX~rSAC=yJ|u07b*niqo4Z95RnuJQ>`Swc$=z=9L!Q#*CrbkGq)3J?

R3EIMLf)M3J&C_cXv_DiIV&t{bV8xpj~F zK!fP_BPdD%)aQ0R-n-TCpxz3@?}LdnQ;4LRX@ZIi&MEH-s=Z9<=A)#BtwR|i$}gx> zo>JF5iPpJyh#M>DwDgCf`s)p!6r1o*hrB)E1lUJ@S#>*Ci1c&01^7~5-=u^|lCR`a zM;RgG-%N7q7lT?Wj35~EQ=aQ-ABDA9nou2I{wulPN~E!Z8GnSuMW@}dQ19OO5Wx=e zues}4;MH7wPK$RYOZ<@BJWn_{dS3bCz0H1n{#QT5lY3} zhS$4W6|Z)HK$$6+seMgEGB3{$mzEt9M^9r#0<($3d!<#I>e`6W^D$ir6z{{Dc6*nf zmLAV9esycL$hV^uaI$W@T`itLI=~08b)qI8?i; zI=xqj%0~H_bAbAkx47(R$gBb+Ry#>-0qQVQ`pXkA@MxA_5Q|#&em*%auY-;N3L<)=&^(OGh##t>9r6kQaV$k?*qd9~&GV$uCDm6W30AaOh}~(Q zqA56#FnD8#p1ODdA_CtC2E`Cfr)T1uASWmAPdpy4rO%`mk7qK;%2`8l2LypE7b@;l+k?U4o) z^yB36&W)u@HfBb++_%{kD+ypC3s6sLk4*g-{1Hj-r+B*|#(@pr z6C^XoFB{wll#fDJ@W{?_2u+SosggO9WJJ4dpYx5L{!`^|vM z;QroOLQ;;u6e#S9p>ZLDrjQ>}=6(R?inog2szHR2AlON@L?P1Nxvuf(GiIm4w-DZE zLGfi?9``@YL{T8@+*w?lSQvja3OP2+LK^(-W=q$9s}U&x++B*#R&?{#^Zg_&Ny1P6 zv2oiFjV&BN;P~^%AL;&|fwfY6y1HA!f$<)^s8O70=+F5ZkWMOQYBs=Q0il~s9_9;) zxPYyI#W$GW`{6!;2qaxF^u(a!X#$oh*(;9T1ivi+xzh#HNct4sO-D~?K1CuVvkyVP zvYrf#-P+_zq5xXiH_chOf3lv2S6`z%FuT&YxQJ+fe^n_nMLPtnv!-#+8~;1ROQX4| z@os2W9Q#;;HC)yY)};&YzdI9(=7Qg%jpbtX^UInS4-kp08Ig4jYxgScZdd#OsA(R| zZo*`ffo~*v$0G7TmDJ6jW=P+oid4Y17@Mh$;{m&ApeUcorw@@(oQ#dJNJWCmCLjeZ2@K>A@IeB6Q0rJB8WV6Q z$tGIJo?00Q3zMHUh!fvsw-T^h9Qd35sywK|k5>z+kO*4UJ{?N@eK=V2;#U)I`?DX^ymeTQ)<3zc+zrIyC zm96#tW*L~+<_yp7;;YvlT1gs`ZbposhBdnc{y;?;j8>*h8_^eun4(O|*pg>jrk+3? z{WX#ViMPOSCsis@(*AWY0O5-KuCft2jqdTF78%TA&FEZ zf%Xlv87S+3{w&l^0-NP6UZ^WM5Q*9Ngl2>*8!JuNDT9BCBI~0 zY&*7w8*f5TYg(^o9mynO^U*2}jA*({oM}V~NH*q53z3eJEV{Y;`^=*F5bQV%h{ zgRna3&AOi9K)>^;y`qqH)65)3Uj4KQtn-%{jHRCwEi1WID4*0|2t!*108ZApR|a*D zn?I!hAA_Y=9{qhSQv=iB4+Nr9-&`GI+-+-_S+I)8dYp{di$g}aBj5eKq9-PgrYKJ8 zdK2(p>pwr4so&CkQx-<5SN@jZ-c%GsstxUMr-;19!9t5zizk17d62TQfiSoGKb&N0 zGUMkcFPL-3Jh8xwnfU>OtrOm(VkABJtFxZ+S8C7Y&=dW-|fb6`~?PnJ5t? zr{l@Qe&K+7Y)fCBAKlt-{_y+1*SP%t`|SQ!tC2e3>-ykP;%7p~Y49XyuUe_j!3L~LOeC1HL=zM;IyrmO3da#EW~EYmD!K3a z8C*eNJkl#jXnSRG#N#~_Fj3=w8nZb9Yc0T`-jzX}$X~rJ08r!o+2MwA?dK$i zR)DNp6>FzSZzsSL;GZeqrU3sejk9+x!T;r>(R_3iMH#6zj7H0-fTFF2u0LN(kNWPX zw-bSA0ZP*~E((QUFTL%4lPC+tqnndAZ;kM(BEHX1Hu=^NCA0xH3$w%?-`i@-keH{*~i2Ot;-&2#W z!0Y8Zyn00*A&~;UBB8&kSehD@MElsr$Ivx#Gzrvgb*j-3;6D~;p6k|ZzT&%sV>h+? zpaM9n>hgLVj$6ZD;wa)o%4qQUOzo%1)psoaQ~^2~jqSx^*>8%&J}PRpN3XciR`fOe z(R6PeRcN$my$CNF9q!La(rkO8B)Kv``19qnJGsApe(T1KzYY&CKwG$;jZFY;V|S+h z(89umgt-hmZwso$xhFDPKX*69`0o%ZFez5G(goxFIZokQA5Wm-o`=1|DB48^U;;|jIzNM z*aWSaTLFc#D^_ZMC@J2`=i2Wd9n$JiB=O_;`z4?6SH8DBe7N)A>m5&aaBvh9g2i{y z+vx*|Q2gfZvv0OR1{}Dxgy)T~zwl3@RGf2oZ3}~a?x!G8?@lhZ zz3bin7+yT^{zOE(3syN|?#8=l-1M;uKtSxZUV<#6qpLS=`gn!Ea?(?`LI%D4Dcdf8 z`IMt%b@3}sW0-n z_g^8Ww|UVKRjr7bc50000 - - + - - + Go Datatable (Serverside) - + + + + + - Go Datatable (Serverside) + + + + +

+ CMS +

+ RUCIO DATASETS MONITORING +

+
-
-

Go Datatable Serverside

+
- - + + + - - + + + + + + +
Id RseType Dataset LastAccess LastAccessMsMaxDatasetSizeInRsesGB(MaxDatasetSizeInRsesGB MinDatasetSizeInRsesGBAvgDatasetSizeInRsesGB SumDatasetSizeInRsesGB RSEs
RseType DatasetRseSizeLastAccessLastAccessMsMaxDatasetSizeInRsesGBMinDatasetSizeInRsesGBAvgDatasetSizeInRsesGBSumDatasetSizeInRsesGBRSEs
- - - - - - - - - - - - + + + + + + + + + + + diff --git a/src/go/rucio-dataset-mon-go/utils/utils.go b/src/go/rucio-dataset-mon-go/utils/utils.go new file mode 100644 index 00000000..fe986a1e --- /dev/null +++ b/src/go/rucio-dataset-mon-go/utils/utils.go @@ -0,0 +1,54 @@ +package utils + +import ( + "github.com/gin-gonic/gin" + "log" + "net/http" + "strings" +) + +// ErrorResponseStruct custom response struct, used in case of error +type ErrorResponseStruct struct { + Status int `json:"status"` + Message string `json:"message"` + Data map[string]string `json:"data"` +} + +// MiddlewareReqHandler handles CORS and HTTP request settings for the context router +func MiddlewareReqHandler() gin.HandlerFunc { + return func(c *gin.Context) { + //c.Writer.Header().Set("Content-Type", "application/json") + c.Writer.Header().Set("Access-Control-Allow-Origin", "*") + c.Writer.Header().Set("Access-Control-Allow-Credentials", "true") + c.Writer.Header().Set("Access-Control-Allow-Headers", "Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization, accept, origin, Cache-Control, X-Requested-With") + c.Writer.Header().Set("Access-Control-Allow-Methods", "POST, OPTIONS, GET, PUT, DELETE") + if c.Request.Method == "OPTIONS" { + c.AbortWithStatus(204) + return + } + c.Next() + } +} + +// ErrorResponse returns error response with given msg and error +func ErrorResponse(c *gin.Context, msg string, err error) { + log.Printf("[ERROR] %s %s", msg, err) + c.JSON(http.StatusInternalServerError, + ErrorResponseStruct{ + Status: http.StatusInternalServerError, + Message: msg, + Data: map[string]string{"data": err.Error()}, + }) +} + +// ConvertOrderEnumToMongoInt converts DataTable enums ("asc" and "desc") to Mongo sorting integer definitions (1,-1) +func ConvertOrderEnumToMongoInt(dir string) int { + switch strings.ToLower(dir) { + case "asc": + return 1 + case "desc": + return -1 + default: + return 0 + } +}