Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Several input and output improvements #195

Merged
merged 7 commits into from
Aug 18, 2023
Merged

Several input and output improvements #195

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
5125f8b
nInput_EndPoints: add key when available
nmsaguiar Aug 8, 2023
fb22db6
avoid null on key
nmsaguiar Aug 8, 2023
2a6a57e
nOutput_ES: +includeRE, +excludeRE
nmsaguiar Aug 9, 2023
30f977d
nInput_EndPoints: support for attrTemplate using chUrls/chPorts
nmsaguiar Aug 10, 2023
916ff7f
nInput_JavaGC: adding accExecTimeMs
nmsaguiar Aug 10, 2023
e6c9390
nInput_Shell: +parseYaml
nmsaguiar Aug 10, 2023
3875ff5
nInput_Shell: +cmdEach, +parseYaml
nmsaguiar Aug 10, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions config/inputs.disabled/yaml/10.getIOPerProc.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
input:
name : Input java gc
cron : "*/2 * * * * *"
execFrom: nInput_Shell
execArgs:
cmdEach : "echo key: {{this}} && echo cmd: `cat /proc/{{this}}/cmdline` && cat /proc/{{this}}/io"
cmd : "find /tmp -type f | grep /tmp/hsperfdata_ | sed 's,.*/,,' | sed 's,^,- ,'"
eachParseYaml: true
parseYaml : true
attrTemplate : "Java processes/IO"
32 changes: 21 additions & 11 deletions config/objects/nInput_EndPoints.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,11 +48,13 @@ nInput_EndPoints.prototype.testURL = function(anEntry) {
canDoIt = false;
}

return {
var _res = {
result: canDoIt,
errorMessage: errorMessage,
latencyInMs: lat
};
}
if (isDef(anEntry.key)) _res.key = anEntry.key
return _res
}
};

Expand Down Expand Up @@ -82,11 +84,13 @@ nInput_EndPoints.prototype.testPort = function(anEntry) {
canDoIt = false;
}

return {
var _res = {
result: canDoIt,
errorMessage: errorMessage,
latencyInMs: lat
};
}
if (isDef(anEntry.key)) _res.key = anEntry.key
return _res
}
};

Expand All @@ -103,19 +107,22 @@ nInput_EndPoints.prototype.input = function(scope, args) {
if (isDef(this.params.chUrls)) {
$ch(this.params.chUrls).forEach((k, v) => {
var attr;
if (isUnDef(this.params.attribute)) {
// For legacy compatibility
if (isDef(this.params.attribute)) this.params.attrTemplate = this.params.attribute
if (isUnDef(this.params.attrTemplate)) {
if (isDef(k.key)) attr = k.key;
if (isString(k)) attr = k;
if (isDef(k.attribute)) attr = k.attribute;
} else {
if (isDef(k.attribute))
attr = k.attribute
else
attr = this.params.attribute
attr = this.params.attrTemplate
}

if (isDef(attr)) {
ret[templify(attr, k)] = this.testURL(v);
if (isUnDef(ret[templify(attr, k)])) ret[templify(attr, k)] = []
ret[templify(attr, k)].push(this.testURL(v))
}
});
}
Expand All @@ -130,21 +137,24 @@ nInput_EndPoints.prototype.input = function(scope, args) {
if (isDef(this.params.chPorts)) {
$ch(this.params.chPorts).forEach((k, v) => {
var attr;
if (isUnDef(this.params.attribute)) {
// For legacy compatibility
if (isDef(this.params.attribute)) this.params.attrTemplate = this.params.attribute
if (isUnDef(this.params.attrTemplate)) {
if (isDef(k.key)) attr = k.key;
if (isString(k)) attr = k;
if (isDef(k.attribute)) attr = k.attribute;
} else {
if (isDef(k.attribute))
attr = k.attribute
else
attr = this.params.attribute
attr = this.params.attrTemplate
}

if (isDef(attr)) {
ret[templify(attr, k)] = this.testPort(v);
if (isUnDef(ret[templify(attr, k)])) ret[templify(attr, k)] = []
ret[templify(attr, k)].push(this.testPort(v))
}
});
})
}

return ret;
Expand Down
3 changes: 2 additions & 1 deletion config/objects/nInput_JavaGC.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -195,7 +195,8 @@ nInput_JavaGC.prototype.get = function(keyData, extra) {
totalRunningTimeMs: data.sun.rt.__totalRunningTime,
percAppTime : data.sun.rt.__percAppTime,
gcCause : data.sun.gc.cause,
gcLastCause : data.sun.gc.lastCause
gcLastCause : data.sun.gc.lastCause,
accExecTimeMs : (isDef(data.sun.gc.__collectorsAccTimeMs) ? data.sun.gc.__collectorsAccTimeMs : __)
})

if (isArray(data.sun.gc.collector)) {
Expand Down
158 changes: 97 additions & 61 deletions config/objects/nInput_Shell.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,145 +7,181 @@
* \
* - cmd (the command-line to execute)\
* - parseJson (boolean to indicate if the cmd output is json parsable)\
* - parseYaml (boolean to indicate if the cmd output is yaml parsable)\
* - attrTemplate (a string template for the name of the attribute)\
* - keys (a map with a SSH key or array of maps with SSH keys)\
* - chKeys (a channel with similar maps as keys)\
* \
* </odoc>
*/
var nInput_Shell = function(aCommand, isJson, attributeName) {
if (isObject(aCommand)) {
this.params = aCommand;
this.cmd = (isDef(aCommand.cmd) ? aCommand.cmd : "");
this.parseJson = (isDef(aCommand.parseJson) ? aCommand.parseJson : false);
this.name = (isDef(aCommand.name) ? aCommand.name : this.cmd);
this.attrTemplate = (isDef(aCommand.attrTemplate) ? aCommand.attrTemplate : "Server status/{{name}}");
} else {
this.cmd = aCommand;
this.parseJson = isJson;
this.name = "";
this.attrTemplate = (isUnDef(attributeName)) ? "Server status/" + aCommand : attributeName;
var nInput_Shell = function(params) {
if (isMap(params)) {
this.params = params
this.cmd = (isDef(params.cmd) ? params.cmd : "")
this.cmdEach = (isDef(params.cmdEach) ? params.cmdEach : "")
this.parseJson = (isDef(params.parseJson) ? params.parseJson : false)
this.parseYaml = (isDef(params.parseYaml) ? params.parseYaml : false)
this.eachParseJson = (isDef(params.eachParseJson) ? params.eachParseJson : false)
this.eachParseYaml = (isDef(params.eachParseYaml) ? params.eachParseYaml : false)
this.name = (isDef(params.name) ? params.name : this.cmd)
this.attrTemplate = (isDef(params.attrTemplate) ? params.attrTemplate : "Shell/{{name}}")
}

nInput.call(this, this.input);
nInput.call(this, this.input)
};
inherit(nInput_Shell, nInput);
inherit(nInput_Shell, nInput)

/**
*/
nInput_Shell.prototype.input = function(scope, args) {
var ret = {};
var ret = {}

// Generic parsing functions
var _parse = s => {
if (this.params.parseJson) return jsonParse(s, true)
if (this.params.parseYaml) return af.fromYAML(s)
}
var _parseEach = s => {
if (this.params.eachParseJson) return jsonParse(s, true)
if (this.params.eachParseYaml) return af.fromYAML(s)
}

// Generic execution functions
var _exec = (fn, cmd, cmdEach) => {
if (isDef(cmdEach)) {
var lst = _parse(fn(cmd))
if (isArray(lst)) {
var _r = []
lst.forEach(v => {
lprint(templify(cmdEach, v))
var __r = _parseEach(fn(templify(cmdEach, v)))
_r.push(__r)
})
} else {
throw "nInput_Shell | Not a list/array (" + af.toSLON(lst) + ")"
}
return _r
} else {
return _parse(fn(cmd))
}
}

if (isDef(this.params.chKeys) || isDef(this.params.keys)) {
var res = [], attrname;
var parent = this;
var res = [], attrname
var parent = this

if (isArray(this.cmd)) this.cmd = this.cmd.join(" && ");
_$(this.cmd).isString().$_();
if (isArray(this.cmd)) this.cmd = this.cmd.join(" && ")
_$(this.cmd).isString().$_()

if (isDef(this.params.chKeys)) this.params.keys = $ch(this.params.chKeys).getKeys().map(r => r.key);
if (isDef(this.params.chKeys)) this.params.keys = $ch(this.params.chKeys).getKeys().map(r => r.key)

for(var i in this.params.keys) {
var v = $ch(this.params.chKeys).get({ key: this.params.keys[i] });
v = __nam_getSec(v);
var v = $ch(this.params.chKeys).get({ key: this.params.keys[i] })
v = __nam_getSec(v)

try {
switch(v.type) {
case "kube":
if (isUnDef(getOPackPath("Kube"))) {
throw "Kube opack not installed.";
throw "Kube opack not installed."
}

var s = $sec(v.secRepo, v.secBucket, v.secBucketPass, v.secMainPass, v.secFile);
var s = $sec(v.secRepo, v.secBucket, v.secBucketPass, v.secMainPass, v.secFile)
var k;
if (isDef(v.secObjKey)) {
var k = s.getObj(v.secObjKey);
var k = s.getObj(v.secObjKey)
}
if (isDef(v.secKey)) {
var ka = s.get(v.secKey);
k = new Kube(ka.url, ka.user, ka.pass, ka.wsTimeout, ka.token);
var ka = s.get(v.secKey)
k = new Kube(ka.url, ka.user, ka.pass, ka.wsTimeout, ka.token)
}
if (isUnDef(k) || isUnDef(k.getNamespaces)) {
throw "The secObjKey = '" + v.secObjKey + "' is not a valid Kube object.";
throw "The secObjKey = '" + v.secObjKey + "' is not a valid Kube object."
}

var epods = [];
var epods = []
if (isUnDef(v.pod)) {
if (isDef(v.podTemplate)) {
var pods = k.getPods(v.namespace);
var pods = k.getPods(v.namespace)
epods = $from(pods)
.equals("Kind", "Pod")
.match("Metadata.Name", v.podTemplate)
.select(r => r.Metadata.Name);
.select(r => r.Metadata.Name)
} else {
throw "No pod determined for '" + v.secObjKey + "'";
throw "No pod determined for '" + v.secObjKey + "'"
}
} else {
epods = [ v.pod ];
epods = [ v.pod ]
}

epods.forEach(pod => {
try {
var rr = String(k.exec(v.namespace, pod, [ templify(parent.cmd) ], void 0, true));
if (parent.parseJson) {
//var rr = String(k.exec(v.namespace, pod, [ templify(parent.cmd) ], __, true))
if (parent.parseJson || parent.parseYaml) {
res.push({
key: parent.params.keys[i],
result: jsonParse(rr, true)
result: _exec(c => {
return String(k.exec(v.namespace, pod, [ templify(c) ], __, true))
}, parent.cmd, parent.cmdEach)
});
} else {
res.push({
key: parent.params.keys[i],
result: rr
result: _exec(c => {
return String(k.exec(v.namespace, pod, [ templify(c) ], __, true))
}, parent.cmd, parent.cmdEach)
});
}
;
} catch(e) {
logErr("nInput_Shell | Error on namespace '"+ v.namespace + "', pod '" + pod + "': " + String(e));
logErr("nInput_Shell | Error on namespace '"+ v.namespace + "', pod '" + pod + "': " + String(e))
}
});
})

break;
break
case "ssh":
default:
nattrmon.useObject(this.params.keys[i], (ssh) => {
if (this.parseJson) {
if (this.parseJson || this.parseYaml) {
res.push({
key: this.params.keys[i],
result: jsonParse(ssh.exec(templify(this.cmd)), true)
});
result: _exec(c => {
return ssh.exec(templify(c))
}, this.cmd, this.cmdEach)
})
} else {
res.push({
key: this.params.keys[i],
result: ssh.exec(templify(this.cmd))
});
result: _exec(c => {
return ssh.exec(templify(c))
}, this.cmd, this.cmdEach)
})
}
});
})
}
} catch(pke) {
logErr(" nInput_Shell | " + this.params.keys[i] + " | " + pke)
}
}

if (this.params.keys.length == 1) {
attrname = templify(this.attrTemplate, { name: this.name, key: this.params.keys[0]});
res = res[0].result;
attrname = templify(this.attrTemplate, { name: this.name, key: this.params.keys[0]})
res = res[0].result
} else {
attrname = templify(this.attrTemplate, { name: this.name });
attrname = templify(this.attrTemplate, { name: this.name })
}

ret[attrname] = res;
ret[attrname] = res
} else {
var attrname = templify(this.attrTemplate, { name: this.name });
var value = $sh(templify(this.cmd)).get(0);

value = (isDef(value.stdin) ? value.stdin : "") + (isDef(value.stdout) ? value.stdout : "");

if (this.parseJson) {
ret[attrname] = jsonParse(value, true);
} else {
ret[attrname] = value;
var attrname = templify(this.attrTemplate, { name: this.name })
var fn = c => {
var value = $sh(templify(c)).get(0)
value = (isDef(value.stdin) ? value.stdin : "") + (isDef(value.stdout) ? value.stdout : "")
return value
}

ret[attrname] = _exec(fn, this.cmd, this.cmdEach)
}

return ret;
};
return ret
}
12 changes: 9 additions & 3 deletions config/objects/nOutput_ES.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,16 @@

this.include = aMap.include;
this.exclude = aMap.exclude;
this.includeRE = aMap.includeRE;
this.excludeRE = aMap.excludeRE;

this.unique = aMap.unique;
this.noDateDocId = _$(aMap.noDateDocId, "noDateDocId").isBoolean().default(false)

if (isDef(this.include) && !isArray(this.include)) throw "Include needs to be an array";
if (isDef(this.exclude) && !isArray(this.exclude)) throw "Exclude needs to be an array";
if (isDef(this.includeRE) && !isArray(this.includeRE)) throw "IncludeRE needs to be an array";
if (isDef(this.excludeRE) && !isArray(this.excludeRE)) throw "ExcludeRE needs to be an array";

this.considerSetAll = (isDef(aMap.considerSetAll)) ? aMap.considerSetAll : true;
this.stampMap = aMap.stampMap;
Expand Down Expand Up @@ -188,12 +192,14 @@ nOutput_ES.prototype.output = function (scope, args) {

for (var vi in v) {
var value = v[vi];
var isok = isDef(this.include) ? false : true;
var isok = (isDef(this.include) || isDef(this.includeRE)) ? false : true
var isWarns = (args.ch == "nattrmon::warnings" || args.ch == "nattrmon::warnings::buffer");
var kk = (isWarns) ? v[vi].title : v[vi].name;

if (isDef(this.include) && this.include.indexOf(kk) >= 0) isok = true;
if (isDef(this.exclude) && this.exclude.indexOf(kk) >= 0) isok = false;
if (isDef(this.include) && this.include.indexOf(kk) >= 0) isok = true
if (isDef(this.exclude) && this.exclude.indexOf(kk) >= 0) isok = false
if (isDef(this.includeRE) && kk.match(new RegExp(this.includeRE))) isok = true
if (isDef(this.excludeRE) && kk.match(new RegExp(this.excludeRE))) isok = false
if (isok) { this.addToES($ch(this.ch), value, isWarns); }
}
};