Elastic: devenv: improved devenv-image (#49002)

* devenv: elastic: improved config

* better ascii-codes
This commit is contained in:
Gábor Farkas 2022-05-19 08:29:23 +02:00 committed by GitHub
parent 35ea67c210
commit a3c5834594
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 168 additions and 13 deletions

View File

@ -114,6 +114,18 @@ datasources:
tsdbResolution: 1
tsdbVersion: 3
- name: gdev-elasticsearch
type: elasticsearch
access: proxy
database: "[logs-]YYYY.MM.DD"
url: http://localhost:9200
jsonData:
interval: Daily
timeField: "@timestamp"
logLevelField: level
logMessageField: line
esVersion: 8.0.0
- name: gdev-elasticsearch-v7-metrics
type: elasticsearch
access: proxy

View File

@ -0,0 +1 @@
elastic_version=8.2.0

View File

@ -0,0 +1,3 @@
FROM node:16-alpine
COPY data.js /home/node/data.js

View File

@ -0,0 +1,137 @@
const http = require('http');
if (process.argv.length !== 3) {
throw new Error('invalid command line: use node sendLogs.js ELASTIC_BASE_URL');
}
const ELASTIC_BASE_URL = process.argv[2];
// helper function, do a http request
async function jsonRequest(data, method, url, expectedStatusCode) {
return new Promise((resolve, reject) => {
const req = http.request(
{
protocol: url.protocol,
host: url.hostname,
port: url.port,
path: `${url.pathname}${url.search}`,
method,
headers: { 'content-type': 'application/json' },
},
(res) => {
if (res.statusCode !== expectedStatusCode) {
reject(new Error(`Invalid response: ${res.statusCode}`));
} else {
resolve();
}
}
);
req.on('error', (err) => reject(err));
req.write(JSON.stringify(data));
req.end();
});
}
// helper function, choose a random element from an array
function chooseRandomElement(items) {
const index = Math.trunc(Math.random() * items.length);
return items[index];
}
// helper function, sleep for a duration
async function sleep(duration) {
return new Promise((resolve) => {
setTimeout(resolve, duration);
});
}
async function elasticSendLogItem(timestamp, item) {
// we need the YYYY.MM.DD format
const timestampText = timestamp.toISOString().slice(0, 10).replace(/-/g, '.');
const url = new URL(ELASTIC_BASE_URL);
url.pathname = `/logs-${timestampText}/_doc`;
await jsonRequest(item, 'POST', url, 201);
console.log(`posted to ${url.toString()}`);
}
async function elasticSetupIndexTemplate() {
const data = {
index_patterns: ['logs-*'],
template: {
mappings: {
properties: {
'@timestamp': {
type: 'date',
},
counter: {
type: 'integer',
},
float: {
type: 'float',
},
level: {
type: 'keyword',
},
label: {
type: 'keyword',
},
location: {
type: 'geo_point',
},
},
},
},
};
const url = new URL(ELASTIC_BASE_URL);
url.pathname = '/_index_template/gdev';
await jsonRequest(data, 'PUT', url, 200);
}
function makeRandomPoint() {
const angle = Math.random() * 2 * Math.PI;
const x = 45 * Math.sin(angle);
const y = 45 * Math.cos(angle);
return y + ', ' + x;
}
function getRandomLogItem(counter, timestamp) {
const randomText = `${Math.trunc(Math.random() * 1000 * 1000 * 1000)}`;
const maybeAnsiText = Math.random() < 0.5 ? 'with ANSI \u001b[31mpart of the text\u001b[0m' : '';
return {
'@timestamp': timestamp.toISOString(),
line: `log text ${maybeAnsiText} [${randomText}]`,
counter: counter.toString(),
float: 100 * Math.random().toString(),
label: chooseRandomElement(['val1', 'val2', 'val3']),
level: chooseRandomElement(['info', 'info', 'error']),
// location: chooseRandomElement(LOCATIONS),
location: makeRandomPoint(),
};
}
async function main() {
await elasticSetupIndexTemplate();
const SLEEP_ANGLE_STEP = Math.PI / 200;
let sleepAngle = 0;
function getNextSineWaveSleepDuration() {
sleepAngle += SLEEP_ANGLE_STEP;
return Math.trunc(1000 * Math.abs(Math.sin(sleepAngle)));
}
for (let step = 0; step < 300; step++) {
await sleep(getNextSineWaveSleepDuration());
const timestamp = new Date();
const item = getRandomLogItem(step + 1, timestamp);
elasticSendLogItem(timestamp, item);
}
}
// when running in docker, we catch the needed stop-signal, to shutdown fast
process.on('SIGTERM', () => {
console.log('shutdown requested');
process.exit(0);
});
main();

View File

@ -1,22 +1,24 @@
# You need to run 'sysctl -w vm.max_map_count=262144' on the host machine
elasticsearch-latest:
image: docker.elastic.co/elasticsearch/elasticsearch:8.0.0-beta1
elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:${elastic_version}
command: elasticsearch
environment:
- "discovery.type=single-node"
- "xpack.license.self_generated.type=basic"
- "xpack.security.enabled=false"
ports:
- "14200:9200"
- "14300:9300"
- "9200:9200"
# TODO: uncomment when https://github.com/grafana/fake-data-gen/pull/20 is merged
# fake-elastic-latest-data:
# image: grafana/fake-data-gen
# links:
# - elasticsearch-latest
# environment:
# FD_SERVER: elasticsearch-latest
# FD_DATASOURCE: elasticsearch8
# FD_PORT: 9200
data:
build: docker/blocks/elastic/data
command: node /home/node/data.js http://elasticsearch:9200
depends_on:
- elasticsearch
# elastic starts slowly, the first couple start of data.js
# might fail, so we auto-restart it on failure.
deploy:
restart_policy:
condition: on-failure
delay: 5s
max_attempts: 20 # should be enough