Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closes #30. Add the ability to specify custom NameNode name and edits dirs. #40

Merged
merged 1 commit into from
May 7, 2018
Merged

Closes #30. Add the ability to specify custom NameNode name and edits dirs. #40

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 29 additions & 13 deletions dynamometer-infra/src/main/java/com/linkedin/dynamometer/AMOptions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@ class AMOptions {
public static final String DATANODES_PER_CLUSTER_DEFAULT = "1";
public static final String DATANODE_LAUNCH_DELAY_ARG = "datanode_launch_delay";
public static final String DATANODE_LAUNCH_DELAY_DEFAULT = "0s";
public static final String NAMENODE_NAME_DIR_ARG = "namenode_name_dir";
public static final String NAMENODE_EDITS_DIR_ARG = "namenode_edits_dir";

private final int datanodeMemoryMB;
private final int datanodeVirtualCores;
Expand All @@ -51,6 +53,8 @@ class AMOptions {
private final String namenodeArgs;
private final String namenodeNodeLabelExpression;
private final int namenodeMetricsPeriod;
private final String namenodeNameDir;
private final String namenodeEditsDir;
// Original shellEnv as passed in through arguments
private final Map<String, String> originalShellEnv;
// Extended shellEnv including custom environment variables
Expand All @@ -59,7 +63,7 @@ class AMOptions {
AMOptions(int datanodeMemoryMB, int datanodeVirtualCores, String datanodeArgs,
String datanodeNodeLabelExpression, int datanodesPerCluster, String datanodeLaunchDelay, int namenodeMemoryMB,
int namenodeVirtualCores, String namenodeArgs, String namenodeNodeLabelExpression, int namenodeMetricsPeriod,
Map<String, String> shellEnv) {
String namenodeNameDir, String namenodeEditsDir, Map<String, String> shellEnv) {
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this constructor is growing huge and a lot have default values, shall we construct this through a builder?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

My initial thought was to agree with you, but actually, I think it makes sense for it to stay as-is. Since this is just an internal class for encapsulating the result of a number of command-line options, you wouldn't normally call this constructor directly, and in fact the only current usage is within this class (initFromParser()). It does not seem worth creating a builder given this setup.

this.datanodeMemoryMB = datanodeMemoryMB;
this.datanodeVirtualCores = datanodeVirtualCores;
this.datanodeArgs = datanodeArgs;
Expand All @@ -71,12 +75,16 @@ class AMOptions {
this.namenodeArgs = namenodeArgs;
this.namenodeNodeLabelExpression = namenodeNodeLabelExpression;
this.namenodeMetricsPeriod = namenodeMetricsPeriod;
this.namenodeNameDir = namenodeNameDir;
this.namenodeEditsDir = namenodeEditsDir;
this.originalShellEnv = shellEnv;
this.shellEnv = new HashMap<>(this.originalShellEnv);
this.shellEnv.put(DynoConstants.NN_ADDITIONAL_ARGS_ENV, this.namenodeArgs);
this.shellEnv.put(DynoConstants.DN_ADDITIONAL_ARGS_ENV, this.datanodeArgs);
this.shellEnv.put(DynoConstants.NN_FILE_METRIC_PERIOD_ENV,
String.valueOf(this.namenodeMetricsPeriod));
this.shellEnv.put(DynoConstants.NN_NAME_DIR_ENV, this.namenodeNameDir);
this.shellEnv.put(DynoConstants.NN_EDITS_DIR_ENV, this.namenodeEditsDir);
}

/**
Expand Down Expand Up @@ -104,28 +112,28 @@ void verify() throws IllegalArgumentException {
void addToVargs(List<String> vargs) {
vargs.add("--" + DATANODE_MEMORY_MB_ARG + " " + String.valueOf(datanodeMemoryMB));
vargs.add("--" + DATANODE_VCORES_ARG + " " + String.valueOf(datanodeVirtualCores));
if (!datanodeArgs.isEmpty()) {
vargs.add("--" + DATANODE_ARGS_ARG + " \\\"" + datanodeArgs + "\\\"");
}
if (!datanodeNodeLabelExpression.isEmpty()) {
vargs.add("--" + DATANODE_NODELABEL_ARG + " \\\"" + datanodeNodeLabelExpression + "\\\"");
}
addStringValToVargs(vargs, DATANODE_ARGS_ARG, datanodeArgs);
addStringValToVargs(vargs, DATANODE_NODELABEL_ARG, datanodeNodeLabelExpression);
vargs.add("--" + DATANODES_PER_CLUSTER_ARG + " " + String.valueOf(datanodesPerCluster));
vargs.add("--" + DATANODE_LAUNCH_DELAY_ARG + " " + datanodeLaunchDelay);
vargs.add("--" + NAMENODE_MEMORY_MB_ARG + " " + String.valueOf(namenodeMemoryMB));
vargs.add("--" + NAMENODE_VCORES_ARG + " " + String.valueOf(namenodeVirtualCores));
if (!namenodeArgs.isEmpty()) {
vargs.add("--" + NAMENODE_ARGS_ARG + " \\\"" + namenodeArgs + "\\\"");
}
if (!namenodeNodeLabelExpression.isEmpty()) {
vargs.add("--" + NAMENODE_NODELABEL_ARG + " \\\"" + namenodeNodeLabelExpression + "\\\"");
}
addStringValToVargs(vargs, NAMENODE_ARGS_ARG, namenodeArgs);
addStringValToVargs(vargs, NAMENODE_NODELABEL_ARG, namenodeNodeLabelExpression);
vargs.add("--" + NAMENODE_METRICS_PERIOD_ARG + " " + String.valueOf(namenodeMetricsPeriod));
addStringValToVargs(vargs, NAMENODE_NAME_DIR_ARG, namenodeNameDir);
addStringValToVargs(vargs, NAMENODE_EDITS_DIR_ARG, namenodeEditsDir);
for (Map.Entry<String, String> entry : originalShellEnv.entrySet()) {
vargs.add("--" + SHELL_ENV_ARG + " " + entry.getKey() + "=" + entry.getValue());
}
}

private void addStringValToVargs(List<String> vargs, String optionName, String val) {
if (!val.isEmpty()) {
vargs.add("--" + optionName + " \\\"" + val + "\\\"");
}
}

int getDataNodeMemoryMB() {
return datanodeMemoryMB;
}
Expand Down Expand Up @@ -187,6 +195,12 @@ static void setOptions(Options opts) {
"The period in seconds for the NameNode's metrics to be emitted to file; if <=0, " +
"disables this functionality. Otherwise, a metrics file will be stored in the " +
"container logs for the NameNode (default " + NAMENODE_METRICS_PERIOD_DEFAULT + ").");
opts.addOption(NAMENODE_NAME_DIR_ARG, true,
"The directory to use for the NameNode's name data directory. If not specified, a location " +
"within the container's working directory will be used.");
opts.addOption(NAMENODE_EDITS_DIR_ARG, true,
"The directory to use for the NameNode's edits directory. If not specified, a location " +
"within the container's working directory will be used.");
opts.addOption(DATANODE_MEMORY_MB_ARG, true,
"Amount of memory in MB to be requested to run the DNs (default " + DATANODE_MEMORY_MB_DEFAULT + ")");
opts.addOption(DATANODE_VCORES_ARG, true,
Expand Down Expand Up @@ -240,6 +254,8 @@ static AMOptions initFromParser(CommandLine cliParser) {
cliParser.getOptionValue(NAMENODE_ARGS_ARG, ""),
cliParser.getOptionValue(NAMENODE_NODELABEL_ARG, ""),
Integer.parseInt(cliParser.getOptionValue(NAMENODE_METRICS_PERIOD_ARG, NAMENODE_METRICS_PERIOD_DEFAULT)),
cliParser.getOptionValue(NAMENODE_NAME_DIR_ARG, ""),
cliParser.getOptionValue(NAMENODE_EDITS_DIR_ARG, ""),
originalShellEnv);
}

Expand Down
6 changes: 6 additions & 0 deletions dynamometer-infra/src/main/java/com/linkedin/dynamometer/DynoConstants.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,12 @@ public boolean accept(Path path) {
public static final String NN_ADDITIONAL_ARGS_ENV = "NN_ADDITIONAL_ARGS";
// Environment variable which will contain additional arguments for the DataNode
public static final String DN_ADDITIONAL_ARGS_ENV = "DN_ADDITIONAL_ARGS";
// Environment variable which will contain the directory to use for the NameNode's name directory;
// if not specified a directory within the YARN container working directory will be used.
public static final String NN_NAME_DIR_ENV = "NN_NAME_DIR";
// Environment variable which will contain the directory to use for the NameNode's edits directory;
// if not specified a directory within the YARN container working directory will be used.
public static final String NN_EDITS_DIR_ENV = "NN_EDITS_DIR";

public static final String NN_FILE_METRIC_PERIOD_ENV = "NN_FILE_METRIC_PERIOD";

Expand Down
13 changes: 7 additions & 6 deletions dynamometer-infra/src/main/resources/start-component.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -235,11 +235,12 @@ namenode.sink.dyno-file.filename=${nnMetricOutputFileLocal}
EOF
fi

nameDir="$baseDir/name-data"
nameDir=${NN_NAME_DIR:-${baseDir}/name-data}
editsDir=${NN_EDITS_DIR:-${baseDir}/name-data}
checkpointDir="$baseDir/checkpoint"
rm -rf "$nameDir" "$checkpointDir"
mkdir -p "$nameDir/current" "$checkpointDir"
chmod -R 700 "$nameDir" "$checkpointDir"
rm -rf "$nameDir" "$editsDir" "$checkpointDir"
mkdir -p "$nameDir/current" "$editsDir/current" "$checkpointDir"
chmod -R 700 "$nameDir" "$editsDir" "$checkpointDir"
fsImageFile="`ls -1 | grep -E '^fsimage_[[:digit:]]+$' | tail -n 1`"
fsImageMD5File="`ls -1 | grep -E '^fsimage_[[:digit:]]+.md5$' | tail -n 1`"
ln -snf "`pwd`/$fsImageFile" "$nameDir/current/$fsImageFile"
Expand All @@ -256,8 +257,8 @@ EOF
-D dfs.namenode.servicerpc-address=${nnHostname}:${nnServiceRpcPort}
-D dfs.namenode.http-address=${nnHostname}:${nnHttpPort}
-D dfs.namenode.https-address=${nnHostname}:0
-D dfs.namenode.name.dir=file://${baseDir}/name-data
-D dfs.namenode.edits.dir=file://${baseDir}/name-data
-D dfs.namenode.name.dir=file://${nameDir}
-D dfs.namenode.edits.dir=file://${editsDir}
-D dfs.namenode.checkpoint.dir=file://${baseDir}/checkpoint
-D dfs.namenode.kerberos.internal.spnego.principal=
-D dfs.hosts=
Expand Down