IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

configurable docker tags for grafana

Browse Source
This commit is contained in:
Jonathan Shook 2020-05-31 23:58:05 -05:00
parent 98bb5feb4c
commit b248c86684
3 changed files with 107 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java
View File

@ -186,10 +186,15 @@ public class NBCLI {
} }
String reportGraphiteTo = options.wantsReportGraphiteTo(); String reportGraphiteTo = options.wantsReportGraphiteTo();
if (options.wantsDockerMetrics()) { if (options.wantsDockerMetrics()) {
logger.info("Docker metrics is enabled. Docker must be installed for this to work"); logger.info("Docker metrics is enabled. Docker must be installed for this to work");
DockerMetricsManager dmh = new DockerMetricsManager(); DockerMetricsManager dmh = new DockerMetricsManager();
dmh.startMetrics(); Map<String,String> dashboardOptions = Map.of(
DockerMetricsManager.GRAFANA_TAG, options.getDockerGrafanaTag()
);
dmh.startMetrics(dashboardOptions);
String warn = "Docker Containers are started, for grafana and prometheus, hit" + String warn = "Docker Containers are started, for grafana and prometheus, hit" +
" these urls in your browser: http://<host>:3000 and http://<host>:9090"; " these urls in your browser: http://<host>:3000 and http://<host>:9090";
logger.warn(warn); logger.warn(warn);

12
engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIOptions.java
View File

@ -79,6 +79,7 @@ public class NBCLIOptions {
private static final String GRAALJS_ENGINE = "--graaljs"; private static final String GRAALJS_ENGINE = "--graaljs";
private static final String NASHORN_ENGINE = "--nashorn"; private static final String NASHORN_ENGINE = "--nashorn";
private static final String GRAALJS_COMPAT = "--graaljs-compat"; private static final String GRAALJS_COMPAT = "--graaljs-compat";
private static final String DOCKER_GRAFANA_TAG = "--docker-grafana-tag";
public static final Set<String> RESERVED_WORDS = new HashSet<>() {{ public static final Set<String> RESERVED_WORDS = new HashSet<>() {{
@ -103,7 +104,7 @@ public class NBCLIOptions {
private String reportGraphiteTo = null; private String reportGraphiteTo = null;
private String reportCsvTo = null; private String reportCsvTo = null;
private int reportInterval = 10; private int reportInterval = 10;
private String metricsPrefix = "nosqlbench."; private String metricsPrefix = "nosqlbench";
private String wantsMetricsForActivity; private String wantsMetricsForActivity;
private String sessionName = ""; private String sessionName = "";
private boolean showScript = false; private boolean showScript = false;
@ -129,6 +130,7 @@ public class NBCLIOptions {
private Scenario.Engine engine = Scenario.Engine.Graalvm; private Scenario.Engine engine = Scenario.Engine.Graalvm;
private boolean graaljs_compat = false; private boolean graaljs_compat = false;
private int hdr_digits = 4; private int hdr_digits = 4;
private String docker_grafana_tag = "7.0.1";
public NBCLIOptions(String[] args) { public NBCLIOptions(String[] args) {
parse(args); parse(args);
@ -184,6 +186,10 @@ public class NBCLIOptions {
} }
switch (word) { switch (word) {
case DOCKER_GRAFANA_TAG:
arglist.removeFirst();
docker_grafana_tag = readWordOrThrow(arglist,"grafana docker tag");
break;
case GRAALJS_COMPAT: case GRAALJS_COMPAT:
graaljs_compat = true; graaljs_compat = true;
arglist.removeFirst(); arglist.removeFirst();
@ -630,6 +636,10 @@ public class NBCLIOptions {
return wantsWorkloadsList; return wantsWorkloadsList;
} }
public String getDockerGrafanaTag() {
return docker_grafana_tag;
}
public static class LoggerConfig { public static class LoggerConfig {
public String file; public String file;
public String pattern = ".*"; public String pattern = ".*";

123
engine-docker/src/main/java/io/nosqlbench/engine/docker/DockerMetricsManager.java
View File

@ -12,11 +12,13 @@ import com.github.dockerjava.api.model.ContainerNetworkSettings;
import com.github.dockerjava.api.model.Frame; import com.github.dockerjava.api.model.Frame;
import com.github.dockerjava.core.async.ResultCallbackTemplate; import com.github.dockerjava.core.async.ResultCallbackTemplate;
import com.github.dockerjava.core.command.LogContainerResultCallback; import com.github.dockerjava.core.command.LogContainerResultCallback;
import io.nosqlbench.nb.api.content.Content;
import io.nosqlbench.nb.api.content.NBIO; import io.nosqlbench.nb.api.content.NBIO;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.io.*; import java.io.*;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.Paths; import java.nio.file.Paths;
@ -27,6 +29,8 @@ import static io.nosqlbench.engine.docker.RestHelper.post;
public class DockerMetricsManager { public class DockerMetricsManager {
public static final String GRAFANA_TAG = "grafana_tag";
private final DockerHelper dh; private final DockerHelper dh;
String userHome = System.getProperty("user.home"); String userHome = System.getProperty("user.home");
@ -35,22 +39,22 @@ public class DockerMetricsManager {
public DockerMetricsManager() { public DockerMetricsManager() {
dh = new DockerHelper(); dh = new DockerHelper();
} }
public void startMetrics() { public void startMetrics(Map<String, String> options) {
String ip = startGraphite(); String ip = startGraphite();
startPrometheus(ip); startPrometheus(ip);
startGrafana(ip); startGrafana(ip, options.get(GRAFANA_TAG));
} }
private void startGrafana(String ip) { private void startGrafana(String ip, String tag) {
String GRAFANA_IMG = "grafana/grafana"; String GRAFANA_IMG = "grafana/grafana";
String tag = "5.3.2"; tag = (tag == null || tag.isEmpty()) ? "latest" : tag;
String name = "grafana"; String name = "grafana";
List<Integer> port = Arrays.asList(3000); List<Integer> port = Arrays.asList(3000);
@ -60,24 +64,24 @@ public class DockerMetricsManager {
} }
List<String> volumeDescList = Arrays.asList( List<String> volumeDescList = Arrays.asList(
userHome + "/.nosqlbench/grafana:/var/lib/grafana:rw" userHome + "/.nosqlbench/grafana:/var/lib/grafana:rw"
//cwd+"/docker-metrics/grafana:/grafana", //cwd+"/docker-metrics/grafana:/grafana",
//cwd+"/docker-metrics/grafana/datasources:/etc/grafana/provisioning/datasources", //cwd+"/docker-metrics/grafana/datasources:/etc/grafana/provisioning/datasources",
//cwd+"/docker-metrics/grafana/dashboardconf:/etc/grafana/provisioning/dashboards" //cwd+"/docker-metrics/grafana/dashboardconf:/etc/grafana/provisioning/dashboards"
//,cwd+"/docker-metrics/grafana/dashboards:/var/lib/grafana/dashboards:ro" //,cwd+"/docker-metrics/grafana/dashboards:/var/lib/grafana/dashboards:ro"
); );
List<String> envList = Arrays.asList( List<String> envList = Arrays.asList(
"GF_SECURITY_ADMIN_PASSWORD=admin", "GF_SECURITY_ADMIN_PASSWORD=admin",
"GF_AUTH_ANONYMOUS_ENABLED=\"true\"", "GF_AUTH_ANONYMOUS_ENABLED=\"true\"",
"GF_SNAPSHOTS_EXTERNAL_SNAPSHOT_URL=https://assethub.datastax.com:3001", "GF_SNAPSHOTS_EXTERNAL_SNAPSHOT_URL=https://assethub.datastax.com:3001",
"GF_SNAPSHOTS_EXTERNAL_SNAPSHOT_NAME=\"Upload to DataStax\"" "GF_SNAPSHOTS_EXTERNAL_SNAPSHOT_NAME=\"Upload to DataStax\""
); );
String reload = null; String reload = null;
List<String> linkNames = new ArrayList(); List<String> linkNames = new ArrayList();
linkNames.add("prom"); linkNames.add("prom");
String containerId = dh.startDocker(GRAFANA_IMG, tag, name, port, volumeDescList, envList, null, reload, linkNames); String containerId = dh.startDocker(GRAFANA_IMG, tag, name, port, volumeDescList, envList, null, reload, linkNames);
if (containerId == null){ if (containerId == null) {
return; return;
} }
@ -103,19 +107,19 @@ public class DockerMetricsManager {
} }
List<String> volumeDescList = Arrays.asList( List<String> volumeDescList = Arrays.asList(
//cwd+"/docker-metrics/prometheus:/prometheus", //cwd+"/docker-metrics/prometheus:/prometheus",
userHome + "/.nosqlbench/prometheus-conf:/etc/prometheus", userHome + "/.nosqlbench/prometheus-conf:/etc/prometheus",
userHome + "/.nosqlbench/prometheus:/prometheus" userHome + "/.nosqlbench/prometheus:/prometheus"
//"./prometheus/tg_dse.json:/etc/prometheus/tg_dse.json" //"./prometheus/tg_dse.json:/etc/prometheus/tg_dse.json"
); );
List<String> envList = null; List<String> envList = null;
List<String> cmdList = Arrays.asList( List<String> cmdList = Arrays.asList(
"--config.file=/etc/prometheus/prometheus.yml", "--config.file=/etc/prometheus/prometheus.yml",
"--storage.tsdb.path=/prometheus", "--storage.tsdb.path=/prometheus",
"--storage.tsdb.retention=183d", "--storage.tsdb.retention=183d",
"--web.enable-lifecycle" "--web.enable-lifecycle"
); );
@ -136,16 +140,23 @@ public class DockerMetricsManager {
String name = "graphite-exporter"; String name = "graphite-exporter";
//TODO: look into UDP //TODO: look into UDP
List<Integer> port = Arrays.asList(9108, 9109); List<Integer> port = Arrays.asList(9108, 9109);
List<String> volumeDescList = Arrays.asList(); List<String> volumeDescList = new ArrayList<String>();
setupGraphiteFiles(volumeDescList);
List<String> envList = Arrays.asList(); List<String> envList = Arrays.asList();
String reload = null; String reload = null;
List<String> linkNames = new ArrayList(); List<String> linkNames = new ArrayList();
dh.startDocker(GRAPHITE_EXPORTER_IMG, tag, name, port, volumeDescList, envList, null, reload, linkNames);
List<String> cmdOpts = Arrays.asList("--graphite.mapping-config=/tmp/graphite_mapping.conf");
dh.startDocker(GRAPHITE_EXPORTER_IMG, tag, name, port, volumeDescList, envList, cmdOpts, reload, linkNames);
logger.info("graphite exporter container started"); logger.info("graphite exporter container started");
logger.info("searching for graphite exporter container ip"); logger.info("searching for graphite exporter container ip");
ContainerNetworkSettings settings = dh.searchContainer(name, null).getNetworkSettings(); ContainerNetworkSettings settings = dh.searchContainer(name, null).getNetworkSettings();
Map<String, ContainerNetwork> networks = settings.getNetworks(); Map<String, ContainerNetwork> networks = settings.getNetworks();
String ip = null; String ip = null;
@ -157,6 +168,29 @@ public class DockerMetricsManager {
return ip; return ip;
} }
private void setupGraphiteFiles(List<String> volumeDescList) {
String exporterConfig = NBIO.readCharBuffer("docker/graphite/graphite_mapping.conf").toString();
File nosqlbenchdir = new File(userHome, "/.nosqlbench/");
mkdir(nosqlbenchdir);
File graphiteExporterDir = new File(userHome, "/.nosqlbench/graphite-exporter");
mkdir(graphiteExporterDir);
Path mappingPath = Path.of(userHome, ".nosqlbench", "graphite-exporter", "graphite_mapping.conf");
if (!Files.exists(mappingPath)) {
try {
Files.writeString(mappingPath, exporterConfig);
} catch (IOException e) {
throw new RuntimeException("Error writing initial graphite mapping config in " + mappingPath, e);
}
}
volumeDescList.add(mappingPath.toString() + ":/tmp/graphite_mapping.conf");
}
private void setupPromFiles(String ip) { private void setupPromFiles(String ip) {
String datasource = NBIO.readCharBuffer("docker/prometheus/prometheus.yml").toString(); String datasource = NBIO.readCharBuffer("docker/prometheus/prometheus.yml").toString();
@ -213,14 +247,14 @@ public class DockerMetricsManager {
} }
private void mkdir(File dir) { private void mkdir(File dir) {
if(dir.exists()){ if (dir.exists()) {
return; return;
} }
if(! dir.mkdir()){ if (!dir.mkdir()) {
if( dir.canWrite()){ if (dir.canWrite()) {
System.out.println("no write access"); System.out.println("no write access");
} }
if( dir.canRead()){ if (dir.canRead()) {
System.out.println("no read access"); System.out.println("no read access");
} }
System.out.println("Could not create directory " + dir.getPath()); System.out.println("Could not create directory " + dir.getPath());
@ -238,6 +272,7 @@ public class DockerMetricsManager {
} }
return exists; return exists;
} }
private boolean promFilesExist() { private boolean promFilesExist() {
File nosqlbenchDir = new File(userHome, "/.nosqlbench/"); File nosqlbenchDir = new File(userHome, "/.nosqlbench/");
boolean exists = nosqlbenchDir.exists(); boolean exists = nosqlbenchDir.exists();
@ -265,7 +300,7 @@ public class DockerMetricsManager {
Files.setPosixFilePermissions(grafanaDirPath, perms); Files.setPosixFilePermissions(grafanaDirPath, perms);
} catch (IOException e) { } catch (IOException e) {
logger.error("failed to set permissions on grafana directory " + logger.error("failed to set permissions on grafana directory " +
"directory " + userHome + "/.nosqlbench/grafana)"); "directory " + userHome + "/.nosqlbench/grafana)");
e.printStackTrace(); e.printStackTrace();
System.exit(1); System.exit(1);
} }
@ -273,9 +308,31 @@ public class DockerMetricsManager {
private void configureGrafana() { private void configureGrafana() {
post("http://localhost:3000/api/dashboards/db", "docker/dashboards/analysis.json", true, "load analysis dashboard"); List<Content<?>> dashboardContent = NBIO.all().prefix("docker/dashboards").extension(".json").list();
post("http://localhost:3000/api/datasources", "docker/datasources/prometheus-datasource.yaml", true, "configure data source");
logger.warn("default grafana creds are admin/admin"); for (Content<?> content : dashboardContent) {
String dashboardData = content.asString();
post(
"http://localhost:3000/api/dashboards/db",
() -> dashboardData,
true,
"load dashboard from " + content.asPath().toString()
);
}
List<Content<?>> datasources = NBIO.all().prefix("docker/datasources").extension(".yaml").list();
for (Content<?> datasource : datasources) {
String datasourceContent = datasource.asString();
post(
"http://localhost:3000/api/datasources",
() -> datasourceContent,
true,
"configure data source from " + datasource.asPath().toString());
}
logger.warn("default grafana creds are admin/admin");
} }