grafana annotator module

This commit is contained in:
Jonathan Shook 2020-11-16 17:35:16 -06:00
parent 317ffab49c
commit 6c0632fb8d
7 changed files with 409 additions and 151 deletions

View File

@ -2,75 +2,32 @@ package io.nosqlbench.engine.clients.grafana;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import io.nosqlbench.engine.clients.grafana.transfer.Annotation;
import io.nosqlbench.engine.clients.grafana.transfer.GrafanaAnnotation;
import io.nosqlbench.engine.clients.grafana.transfer.Annotations;
import io.nosqlbench.engine.clients.grafana.transfer.ApiTokenRequest;
import java.net.Authenticator;
import java.net.PasswordAuthentication;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.util.Base64;
/**
* @see <a href="https://grafana.com/docs/grafana/latest/http_api/annotations/">Grafana Annotations API Docs</a>
*/
public class GrafanaClient {
private final URI baseuri;
private static final Gson gson = new GsonBuilder().setPrettyPrinting().create();
Authenticator auth = null;
private String username;
private String password;
private final GrafanaClientConfig config;
public GrafanaClient(String baseurl) {
this.baseuri = initURI(baseurl);
public GrafanaClient(GrafanaClientConfig config) {
this.config = config;
}
public void basicAuth(String username, String password) {
this.username = username;
this.password = password;
this.auth = new Authenticator() {
@Override
protected PasswordAuthentication getPasswordAuthentication() {
return new PasswordAuthentication(username, password.toCharArray());
}
};
public GrafanaClient(String baseuri) {
this(new GrafanaClientConfig().setBaseUri(baseuri));
}
private URI initURI(String baseurl) {
try {
URI uri = new URI(baseurl);
String userinfo = uri.getRawUserInfo();
if (userinfo != null) {
String[] unpw = userinfo.split(":");
this.username = unpw[0];
this.password = unpw[1];
uri = new URI(baseurl.replace(userinfo + "@", ""));
}
return uri;
} catch (URISyntaxException e) {
throw new RuntimeException(e);
}
}
private HttpClient getClient() {
HttpClient.Builder cb = HttpClient.newBuilder();
if (this.auth != null) {
cb.authenticator(auth);
}
HttpClient client = cb.build();
return client;
}
private URI makeUri(String pathAndQuery) {
try {
return new URI(this.baseuri.toString() + pathAndQuery);
} catch (URISyntaxException e) {
throw new RuntimeException(e);
}
public GrafanaClientConfig getConfig() {
return config;
}
/**
@ -147,12 +104,11 @@ public class GrafanaClient {
public Annotations findAnnotations(By... by) {
String query = By.fields(by);
HttpRequest.Builder rqb = HttpRequest.newBuilder(makeUri("api/annotations?" + query));
rqb = addAuth(rqb);
HttpRequest.Builder rqb = config.newRequest("api/annotations?" + query);
rqb.setHeader("Content-Type", "application/json");
HttpRequest request = rqb.build();
HttpClient client = getClient();
HttpClient client = config.newClient();
HttpResponse<String> response = null;
try {
@ -196,14 +152,12 @@ public class GrafanaClient {
*
* @return
*/
public Annotation createAnnotation(Annotation annotation) {
HttpClient client = getClient();
HttpRequest.Builder rqb = HttpRequest.newBuilder(makeUri("api/annotations"));
rqb = addAuth(rqb);
public GrafanaAnnotation createAnnotation(GrafanaAnnotation grafanaAnnotation) {
HttpClient client = config.newClient();
HttpRequest.Builder rqb = config.newRequest("api/annotations");
rqb.setHeader("Content-Type", "application/json");
String rqBody = gson.toJson(annotation);
String rqBody = gson.toJson(grafanaAnnotation);
rqb = rqb.POST(HttpRequest.BodyPublishers.ofString(rqBody));
addAuth(rqb);
HttpResponse<String> response = null;
try {
@ -218,22 +172,11 @@ public class GrafanaClient {
}
if (response.statusCode() < 200 || response.statusCode() >= 300) {
throw new RuntimeException("Creating annotation failed with status code " + response.statusCode() + " at " +
"baseurl " + baseuri + ": " + response.body());
"baseuri " + config.getBaseUri() + ": " + response.body());
}
String body = response.body();
Annotation savedAnnotation = gson.fromJson(body, Annotation.class);
return savedAnnotation;
}
private HttpRequest.Builder addAuth(HttpRequest.Builder rqb) {
if (this.username != null && this.password != null) {
rqb = rqb.setHeader("Authorization", encodeBasicAuth(username, password));
}
return rqb;
}
private static String encodeBasicAuth(String username, String password) {
return "Basic " + Base64.getEncoder().encodeToString((username + ":" + password).getBytes());
GrafanaAnnotation savedGrafanaAnnotation = gson.fromJson(body, GrafanaAnnotation.class);
return savedGrafanaAnnotation;
}
/**
@ -265,8 +208,8 @@ public class GrafanaClient {
*
* @return
*/
public Annotation createGraphiteAnnotation() {
return null;
public GrafanaAnnotation createGraphiteAnnotation() {
throw new RuntimeException("unimplemented");
}
/**
@ -299,7 +242,7 @@ public class GrafanaClient {
* }</pre>
*/
public void updateAnnotation() {
throw new RuntimeException("unimplemented");
}
/**
@ -332,7 +275,7 @@ public class GrafanaClient {
* }</pre>
*/
public void patchAnnotation() {
throw new RuntimeException("unimplemented");
}
/**
@ -356,7 +299,36 @@ public class GrafanaClient {
* @param id
*/
public void deleteAnnotation(long id) {
throw new RuntimeException("unimplemented");
}
public ApiToken createApiToken(String name, String role, long ttl) {
ApiTokenRequest r = new ApiTokenRequest(name, role, ttl);
ApiToken token = postToGrafana(r, ApiToken.class, "gen api token");
return token;
}
private <T> T postToGrafana(Object request, Class<? extends T> clazz, String desc) {
HttpRequest rq = config.newJsonPOST("api/auth/keys", request);
HttpClient client = config.newClient();
HttpResponse<String> response = null;
try {
response = client.send(rq, HttpResponse.BodyHandlers.ofString());
} catch (Exception e) {
if (e.getMessage().contains("WWW-Authenticate header missing")) {
throw new RuntimeException("Java HttpClient was not authorized, and it saw no WWW-Authenticate header" +
" in the response, so this is probably Grafana telling you that the auth scheme failed. Normally " +
"this error would be thrown by Java HttpClient:" + e.getMessage());
}
throw new RuntimeException(e);
}
if (response.statusCode() < 200 || response.statusCode() >= 300) {
throw new RuntimeException("Request to grafana failed with status code " + response.statusCode() + "\n" +
" while trying to '" + desc + "'\n at baseuri " + config.getBaseUri() + ": " + response.body());
}
String body = response.body();
T result = gson.fromJson(body, clazz);
return result;
}
}

View File

@ -0,0 +1,159 @@
package io.nosqlbench.engine.clients.grafana;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import java.net.Authenticator;
import java.net.PasswordAuthentication;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.time.Duration;
import java.util.*;
import java.util.function.Supplier;
public class GrafanaClientConfig {
private static final Gson gson = new GsonBuilder().setPrettyPrinting().create();
@JsonProperty("baseuri")
private URI baseUrl;
@JsonProperty("timeoutms")
private int timeoutms;
private final List<Authenticator> authenticators = new ArrayList<>();
// private LinkedHashMap<String,String> headers = new LinkedHashMap<>();
private final List<Supplier<Map<String, String>>> headerSources = new ArrayList<>();
public GrafanaClientConfig() {
}
public void basicAuth(String username, String pw) {
Objects.requireNonNull(username);
String authPw = pw != null ? pw : "";
Authenticator basicAuth = new Authenticator() {
@Override
protected PasswordAuthentication getPasswordAuthentication() {
return new PasswordAuthentication(username, authPw.toCharArray());
}
};
addAuthenticator(basicAuth);
addHeader("Authorization", encodeBasicAuth(username, authPw));
}
public GrafanaClientConfig addAuthenticator(Authenticator authenticator) {
authenticators.add(authenticator);
return this;
}
public GrafanaClientConfig addHeader(String headername, String... headervals) {
String headerVal = String.join(";", Arrays.asList(headervals));
addHeaderSource(() -> Map.of(headername, headerVal));
return this;
}
/**
* Add a dynamic header source to be used for every new request.
* Each source provides a map of new headers. If key or value of any
* entry is null or empty, that entry is skipped. Otherwise, they are
* computed and added to every request anew.
*
* @param headerSource A source of new headers
* @return this GrafanaClientConfig, for method chaining
*/
public GrafanaClientConfig addHeaderSource(Supplier<Map<String, String>> headerSource) {
this.headerSources.add(headerSource);
return this;
}
public LinkedHashMap<String, String> getHeaders() {
LinkedHashMap<String, String> headers = new LinkedHashMap<>();
this.headerSources.forEach(hs -> {
Map<String, String> entries = hs.get();
entries.forEach((k, v) -> {
if (k != null && v != null && !k.isEmpty() && !v.isEmpty()) {
headers.put(k, v);
}
});
});
return headers;
}
public HttpClient newClient() {
HttpClient.Builder cb = HttpClient.newBuilder();
cb.connectTimeout(Duration.ofMillis(timeoutms));
for (Authenticator authenticator : authenticators) {
cb.authenticator(authenticator);
}
HttpClient client = cb.build();
return client;
}
private URI makeUri(String pathAndQuery) {
try {
return new URI(getBaseUri().toString() + pathAndQuery);
} catch (URISyntaxException e) {
throw new RuntimeException(e);
}
}
public HttpRequest.Builder newRequest(String path) {
URI requestUri = makeUri(path);
HttpRequest.Builder rqb = HttpRequest.newBuilder(requestUri);
rqb.timeout(Duration.ofMillis(timeoutms));
getHeaders().forEach(rqb::setHeader);
return rqb;
}
public GrafanaClientConfig setBaseUri(String baseuri) {
try {
URI uri = new URI(baseuri);
String userinfo = uri.getRawUserInfo();
if (userinfo != null) {
String[] unpw = userinfo.split(":");
basicAuth(unpw[0], unpw.length == 2 ? unpw[1] : "");
uri = new URI(baseuri.replace(userinfo + "@", ""));
}
this.baseUrl = uri;
} catch (URISyntaxException e) {
throw new RuntimeException(e);
}
return this;
}
private static String encodeBasicAuth(String username, String password) {
return "Basic " + Base64.getEncoder().encodeToString((username + ":" + password).getBytes());
}
public static GrafanaClientConfig fromJson(CharSequence json) {
GrafanaClientConfig grafanaClientConfig = gson.fromJson(json.toString(), GrafanaClientConfig.class);
return grafanaClientConfig;
}
public URI getBaseUri() {
return baseUrl;
}
public HttpRequest newJsonPOST(String pathAndParams, Object rq) {
HttpRequest.Builder rqb = newRequest(pathAndParams);
String body = gson.toJson(rq);
rqb = rqb.POST(HttpRequest.BodyPublishers.ofString(body));
rqb = rqb.setHeader("Content-Type", "application/json");
return rqb.build();
}
public int getTimeoutms() {
return timeoutms;
}
public void setTimeoutms(int timeoutms) {
this.timeoutms = timeoutms;
}
}

View File

@ -2,5 +2,5 @@ package io.nosqlbench.engine.clients.grafana.transfer;
import java.util.ArrayList;
public class Annotations extends ArrayList<Annotation> {
public class Annotations extends ArrayList<GrafanaAnnotation> {
}

View File

@ -1,10 +1,8 @@
package io.nosqlbench.engine.clients.grafana.transfer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.*;
public class Annotation {
public class GrafanaAnnotation {
private Integer id;
private Integer alertId;
@ -19,7 +17,7 @@ public class Annotation {
private String text;
private String metric;
private String type;
private List<String> tags = new ArrayList<String>();
private Map<String, String> tags = new LinkedHashMap<>();
private Object data;
public Integer getId() {
@ -126,18 +124,14 @@ public class Annotation {
this.type = type;
}
public List<String> getTags() {
public Map<String, String> getTags() {
return tags;
}
public void setTags(List<String> tags) {
public void setTags(Map<String, String> tags) {
this.tags = tags;
}
public void setTags(String tags) {
this.tags = Arrays.asList(tags.split("\\\\s,\\\\s"));
}
public Object getData() {
return data;
}

View File

@ -1,6 +1,6 @@
package io.nosqlbench.engine.clients.grafana;
import io.nosqlbench.engine.clients.grafana.transfer.Annotation;
import io.nosqlbench.engine.clients.grafana.transfer.GrafanaAnnotation;
import io.nosqlbench.engine.clients.grafana.transfer.Annotations;
import org.junit.Ignore;
import org.junit.Test;
@ -12,11 +12,11 @@ public class GrafanaClientTest {
@Ignore
public void testCreateAnnotation() {
GrafanaClient client = new GrafanaClient(testurl);
client.basicAuth("admin", "admin");
Annotation a = new Annotation();
client.getConfig().basicAuth("admin", "admin");
GrafanaAnnotation a = new GrafanaAnnotation();
a.setDashboardId(2);
a.setText("testingAnnotation");
Annotation created = client.createAnnotation(a);
GrafanaAnnotation created = client.createAnnotation(a);
System.out.println(created);
}
@ -24,9 +24,17 @@ public class GrafanaClientTest {
@Ignore
public void testFindAnnotations() {
GrafanaClient client = new GrafanaClient(testurl);
client.basicAuth("admin", "admin");
client.getConfig().basicAuth("admin", "admin");
Annotations annotations = client.findAnnotations(By.id(1));
System.out.println(annotations);
}
@Test
@Ignore
public void testGetApiToken() {
GrafanaClient client = new GrafanaClient(testurl);
client.getConfig().basicAuth("admin", "admin");
ApiToken token = client.createApiToken("nosqlbench", "Admin", Long.MAX_VALUE);
System.out.println(token);
}
}

View File

@ -0,0 +1,37 @@
package io.nosqlbench.engine.core.metrics;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.function.Supplier;
public class GrafanaKeyFileReader implements Supplier<String> {
private final static Logger logger = LogManager.getLogger("ANNOTATORS");
private final Path keyfilePath;
public GrafanaKeyFileReader(String sourcePath) {
this.keyfilePath = Path.of(sourcePath);
}
@Override
public String get() {
if (!Files.exists(keyfilePath)) {
logger.warn("apikeyfile does not exist at '" + keyfilePath.toString());
return null;
} else {
try {
String apikey = Files.readString(keyfilePath, StandardCharsets.UTF_8);
return apikey;
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}
}

View File

@ -1,86 +1,97 @@
package io.nosqlbench.engine.core.metrics;
import io.nosqlbench.engine.clients.grafana.GrafanaClient;
import io.nosqlbench.engine.clients.grafana.transfer.Annotation;
import io.nosqlbench.nb.api.annotation.Annotator;
import io.nosqlbench.nb.api.config.ConfigAware;
import io.nosqlbench.nb.api.config.ConfigModel;
import io.nosqlbench.nb.api.config.MutableConfigModel;
import io.nosqlbench.engine.clients.grafana.GrafanaClientConfig;
import io.nosqlbench.engine.clients.grafana.transfer.GrafanaAnnotation;
import io.nosqlbench.nb.annotations.Service;
import io.nosqlbench.nb.api.annotations.Annotation;
import io.nosqlbench.nb.api.annotations.Annotator;
import io.nosqlbench.nb.api.config.*;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Optional;
import java.util.function.Function;
import java.util.function.Supplier;
@Service(value = Annotator.class, selector = "grafana")
public class GrafanaMetricsAnnotator implements Annotator, ConfigAware {
private final GrafanaClient client;
private final static Logger logger = LogManager.getLogger("ANNOTATORS");
private final static Logger annotationsLog = LogManager.getLogger("ANNOTATIONS");
private OnError onError = OnError.Warn;
public GrafanaMetricsAnnotator(String grafanaBaseUrl) {
this.client = new GrafanaClient(grafanaBaseUrl);
private GrafanaClient client;
private Map<String, String> tags = new LinkedHashMap<>();
public GrafanaMetricsAnnotator() {
}
@Override
public void recordAnnotation(String sessionName, long startEpochMillis, long endEpochMillis, Map<String, String> target, Map<String, String> details) {
public void recordAnnotation(Annotation annotation) {
try {
GrafanaAnnotation ga = new GrafanaAnnotation();
Annotation annotation = new Annotation();
ga.setTime(annotation.getStart());
ga.setTimeEnd(annotation.getEnd());
// Target
annotation.getLabels().forEach((k, v) -> {
ga.getTags().put(k, v);
});
ga.getTags().put("layer", annotation.getLayer().toString());
Optional.ofNullable(target.get("type"))
.ifPresent(annotation::setType);
Map<String, String> labels = annotation.getLabels();
long startAt = startEpochMillis > 0 ? startEpochMillis : System.currentTimeMillis();
annotation.setTime(startAt);
annotation.setTimeEnd(endEpochMillis > 0 ? endEpochMillis : startAt);
Optional.ofNullable(labels.get("alertId"))
.map(Integer::parseInt).ifPresent(ga::setAlertId);
String eTime = target.get("timeEnd");
annotation.setTimeEnd((eTime != null) ? Long.valueOf(eTime) : null);
ga.setData(annotation.toString());
Optional.ofNullable(target.get("id")).map(Integer::valueOf)
.ifPresent(annotation::setId);
annotation.getSession();
Optional.ofNullable(target.get("alertId")).map(Integer::valueOf)
.ifPresent(annotation::setAlertId);
Optional.ofNullable(target.get("dashboardId")).map(Integer::valueOf)
.ifPresent(annotation::setDashboardId);
// Target
Optional.ofNullable(labels.get("type"))
.ifPresent(ga::setType);
Optional.ofNullable(target.get("panelId")).map(Integer::valueOf)
.ifPresent(annotation::setPanelId);
Optional.ofNullable(labels.get("id")).map(Integer::valueOf)
.ifPresent(ga::setId);
Optional.ofNullable(target.get("userId")).map(Integer::valueOf)
.ifPresent(annotation::setUserId);
Optional.ofNullable(labels.get("alertId")).map(Integer::valueOf)
.ifPresent(ga::setAlertId);
Optional.ofNullable(target.get("userName"))
.ifPresent(annotation::setUserName);
Optional.ofNullable(labels.get("dashboardId")).map(Integer::valueOf)
.ifPresent(ga::setDashboardId);
Optional.ofNullable(target.get("tags"))
.ifPresent(annotation::setTags);
Optional.ofNullable(labels.get("panelId")).map(Integer::valueOf)
.ifPresent(ga::setPanelId);
Optional.ofNullable(details.get("metric"))
.ifPresent(annotation::setMetric);
Optional.ofNullable(labels.get("userId")).map(Integer::valueOf)
.ifPresent(ga::setUserId);
// Details
Optional.ofNullable(labels.get("userName"))
.ifPresent(ga::setUserName);
StringBuilder sb = new StringBuilder();
if (details.containsKey("text")) {
annotation.setText(details.get("text"));
} else {
for (String dkey : details.keySet()) {
sb.append(sb).append(": ").append(details.get(dkey)).append("\n");
Optional.ofNullable(labels.get("metric"))
.ifPresent(ga::setMetric);
// Details
annotationsLog.info("ANNOTATION:" + ga.toString());
GrafanaAnnotation created = this.client.createAnnotation(ga);
} catch (Exception e) {
switch (onError) {
case Warn:
logger.warn("Error while reporting annotation: " + e.getMessage(), e);
break;
case Throw:
throw e;
}
annotation.setText(details.toString());
}
Optional.ofNullable(details.get("data"))
.ifPresent(annotation::setData);
Optional.ofNullable(details.get("prevState"))
.ifPresent(annotation::setPrevState);
Optional.ofNullable(details.get("newState"))
.ifPresent(annotation::setNewState);
Annotation created = this.client.createAnnotation(annotation);
}
@Override
@ -89,14 +100,91 @@ public class GrafanaMetricsAnnotator implements Annotator, ConfigAware {
}
@Override
public void applyConfig(Map<String, ?> element) {
public void applyConfig(Map<String, ?> providedConfig) {
ConfigModel configModel = getConfigModel();
ConfigReader cfg = configModel.apply(providedConfig);
GrafanaClientConfig gc = new GrafanaClientConfig();
gc.setBaseUri(cfg.param("baseurl", String.class));
if (cfg.containsKey("tags")) {
this.tags = ParamsParser.parse(cfg.param("tags", String.class), false);
}
if (cfg.containsKey("apikeyfile")) {
String apikeyfile = cfg.paramEnv("apikeyfile", String.class);
AuthWrapper authHeaderSupplier = new AuthWrapper(
"Authorization",
new GrafanaKeyFileReader(apikeyfile),
s -> "Bearer " + s + ";"
);
gc.addHeaderSource(authHeaderSupplier);
}
if (cfg.containsKey("apikey")) {
gc.addHeaderSource(() -> Map.of("Authorization", "Bearer " + cfg.param("apikey", String.class)));
}
if (cfg.containsKey("username")) {
if (cfg.containsKey("password")) {
gc.basicAuth(
cfg.param("username", String.class),
cfg.param("password", String.class)
);
} else {
gc.basicAuth(cfg.param("username", String.class), "");
}
}
this.onError = OnError.valueOfName(cfg.get("onerror").toString());
this.client = new GrafanaClient(gc);
}
@Override
public ConfigModel getConfigModel() {
return new MutableConfigModel().add("baseurl", String.class).asReadOnly();
return new MutableConfigModel(this)
.required("baseurl", String.class,
"The base url of the grafana node, like http://localhost:3000/")
.defaultto("apikeyfile", "$NBSTATEDIR/grafana_key",
"The file that contains the api key, supersedes apikey")
.optional("apikey", String.class,
"The api key to use, supersedes basic username and password")
.optional("username", String.class,
"The username to use for basic auth")
.optional("password", String.class,
"The password to use for basic auth")
.defaultto("tags", "source:nosqlbench",
"The tags that identify the annotations, in k:v,... form")
// .defaultto("onerror", OnError.Warn)
.defaultto("onerror", "warn",
"What to do when an error occurs while posting an annotation")
.defaultto("timeoutms", 5000,
"connect and transport timeout for the HTTP client")
.asReadOnly();
}
public static class AuthWrapper implements Supplier<Map<String, String>> {
private final Function<String, String> valueMapper;
private final String headerName;
private final Supplier<String> valueSupplier;
public AuthWrapper(String headerName, Supplier<String> valueSupplier, Function<String, String> valueMapper) {
this.headerName = headerName;
this.valueSupplier = valueSupplier;
this.valueMapper = valueMapper;
}
@Override
public Map<String, String> get() {
String value = valueSupplier.get();
if (value != null) {
value = valueMapper.apply(value);
return Map.of(headerName, value);
}
return Map.of();
}
}
}