feat(telemetry): export traces data in otlp format & matrics data to prometheus (#1400)

Reviewed-on: daoyoucloud/tachybase#1400
Reviewed-by: sealday <zhanglin@daoyoucloud.com>
Co-authored-by: TomyJan <TomyJan6@gmail.com>
Co-committed-by: TomyJan <TomyJan6@gmail.com>
This commit is contained in:
TomyJan 2024-07-31 02:31:08 +08:00 committed by sealday
parent 9261719589
commit ba9dc69086
27 changed files with 1206 additions and 272 deletions

View File

@ -62,3 +62,14 @@ INIT_ROOT_EMAIL=admin@tachybase.com
INIT_ROOT_PASSWORD=!Admin123.
INIT_ROOT_NICKNAME=Super Admin
INIT_ROOT_USERNAME=tachybase
################# TELEMETRY #################
# TELEMETRY_ENABLED=on # 是否启用遥测
# TELEMETRY_SERVICE_NAME=tachybase-default-demo # 服务名称,默认为 tachybase-main
# OTEL_LOG_LEVEL=debug # OpenTelemetry 日志级别,仅输出到控制台,见 [官方文档](https://open-telemetry.github.io/opentelemetry-js/enums/_opentelemetry_api.DiagLogLevel.html) ,不配置则默认不输出日志
# OTEL_METRICS_READER=console,prometheus # 指标读取器两个均为内置console 为控制台prometheus 为 prometheus 服务器
# OTEL_PROMETHEUS_SERVER=on # 是否启用 Prometheus 服务,用于创建服务器以供导出指标数据
# OTEL_PROMETHEUS_PORT=9464 # Prometheus 服务器端口,默认为 9464注意此服务器没有鉴权
# OTEL_TRACES_PROCESSOR=console,otlp # 链路追踪处理器两个均为内置console 为控制台otlp 为 OTLP 规范的追踪服务器
# OTEL_EXPORTER_OTLP_TRACES_ENDPOINT_GRPC=http://localhost:4317 # [OTLP](https://opentelemetry.io/docs/specs/otlp/) 规范的追踪服务器的 gRPC 端点地址

View File

@ -3,9 +3,9 @@ import { AppTelemetryOptions } from '@tachybase/server';
export const telemetry: AppTelemetryOptions = {
enabled: process.env.TELEMETRY_ENABLED === 'on',
metric: {
readerName: process.env.TELEMETRY_METRIC_READER,
readerName: process.env.OTEL_METRICS_READER,
},
trace: {
processorName: process.env.TELEMETRY_TRACE_PROCESSOR,
processorName: process.env.OTEL_TRACES_PROCESSOR,
},
};

View File

@ -1066,8 +1066,13 @@ export class Application<StateT = DefaultState, ContextT = DefaultContext> exten
plugins: plugins || [],
});
let serviceName = process.env.TELEMETRY_SERVICE_NAME;
if (!serviceName) {
this.log.warn('TELEMETRY_SERVICE_NAME is not set, will use default service name, please set it in .env file!');
serviceName = `tachybase-${this.name}`;
}
this._telemetry = new Telemetry({
serviceName: `tachybase-${this.name}`,
serviceName,
version: this.getVersion(),
...options.telemetry,
});

View File

@ -7,12 +7,16 @@
"types": "./lib/index.d.ts",
"dependencies": {
"@opentelemetry/api": "^1.7.0",
"@opentelemetry/auto-instrumentations-node": "^0.48.0",
"@opentelemetry/exporter-prometheus": "^0.52.1",
"@opentelemetry/exporter-trace-otlp-grpc": "^0.52.1",
"@opentelemetry/instrumentation": "^0.46.0",
"@opentelemetry/resources": "^1.19.0",
"@opentelemetry/sdk-metrics": "^1.19.0",
"@opentelemetry/sdk-trace-base": "^1.19.0",
"@opentelemetry/sdk-trace-node": "^1.19.0",
"@opentelemetry/semantic-conventions": "^1.19.0",
"@tachybase/utils": "workspace:*"
"@tachybase/utils": "workspace:*",
"node-os-utils": "^1.3.7"
}
}

View File

@ -2,13 +2,13 @@ import os from 'os';
import osUtils from 'node-os-utils';
export class hardware {
export class environmentMeters {
meter: any;
constructor(meter) {
this.meter = meter;
}
async start() {
console.log('hardware meter start');
console.log('environment meter start');
const uptime = this.meter.createObservableGauge('environment.hardware.uptime', {
description: 'Uptime in seconds',
@ -72,6 +72,6 @@ export class hardware {
});
}
async shutdown() {
console.log('hardware meter shutdown');
console.log('environment meter shutdown');
}
}

View File

@ -0,0 +1,20 @@
import { environmentMeters } from './environment';
import { processMeters } from './process';
export default class initMeters {
meter: any;
constructor(meter) {
this.meter = meter;
}
async start() {
try {
const environmentMeter = new environmentMeters(this.meter);
await environmentMeter.start();
const processMeter = new processMeters(this.meter);
await processMeter.start();
} catch (error) {
console.error('Failed to initialize meters:', error);
}
}
}

View File

@ -1,4 +1,4 @@
export class processSelf {
export class processMeters {
meter: any;
constructor(meter) {
this.meter = meter;
@ -29,6 +29,6 @@ export class processSelf {
});
}
async shutdown() {
console.log('hardware meter shutdown');
console.log('process meter shutdown');
}
}

View File

@ -1,6 +1,7 @@
import { Registry } from '@tachybase/utils';
import opentelemetry from '@opentelemetry/api';
import { PrometheusExporter } from '@opentelemetry/exporter-prometheus';
import { Resource } from '@opentelemetry/resources';
import {
ConsoleMetricExporter,
@ -10,6 +11,8 @@ import {
View,
} from '@opentelemetry/sdk-metrics';
import initMeters from './meters';
export type MetricOptions = {
meterName?: string;
version?: string;
@ -46,6 +49,7 @@ export class Metric {
}
registerReader(name: string, reader: GetMetricReader) {
console.log('register metric reader:', name);
this.readers.register(name, reader);
}
@ -62,6 +66,31 @@ export class Metric {
}
start() {
// 创建 Prometheus 作为指标 Reader
try {
// 启动 Prometheus 服务器
const startServer = process.env.OTEL_PROMETHEUS_SERVER === 'on';
if (!startServer) {
console.warn('Prometheus server is disabled');
} else {
const port = Number(process.env.OTEL_PROMETHEUS_PORT) || 9464;
const reader = () =>
new PrometheusExporter(
{
port, // optional - default is 9464
},
() => {
console.log(`Prometheus exporter endpoint started on http://localhost:${port}/metrics`);
},
);
// 注册 Prometheus 作为指标 Reader
this.registerReader('prometheus', reader);
}
} catch (error) {
console.error('Failed to initialize Prometheus metrics reader:', error);
}
// 添加指标 Reader
let readerName = this.readerName;
if (typeof readerName === 'string') {
readerName = readerName.split(',');
@ -70,6 +99,22 @@ export class Metric {
const reader = this.getReader(name)();
this.provider.addMetricReader(reader);
});
// 初始化指标 Meters
try {
const meter = this.getMeter();
const meters = new initMeters(meter);
meters
.start()
.then(() => {
console.log('Meters initialized');
})
.catch((error) => {
console.error('Failed to initialize meters:', error);
});
} catch (error) {
console.error('Failed to initialize meters:', error);
}
}
shutdown() {

View File

@ -1,3 +1,5 @@
import { diag, DiagConsoleLogger, DiagLogLevel } from '@opentelemetry/api';
import { getNodeAutoInstrumentations } from '@opentelemetry/auto-instrumentations-node';
import { InstrumentationOption, registerInstrumentations } from '@opentelemetry/instrumentation';
import { Resource } from '@opentelemetry/resources';
import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions';
@ -29,6 +31,23 @@ export class Telemetry {
}
init() {
// 设置 OTel 日志等级
const diagLogLevel = process.env.OTEL_LOG_LEVEL;
if (diagLogLevel) {
diag.setLogger(new DiagConsoleLogger(), DiagLogLevel[diagLogLevel]);
}
// 自动插桩
try {
const instrumentations = getNodeAutoInstrumentations();
this.addInstrumentation(instrumentations);
console.log(
`auto-instrumentations added, num: ${instrumentations.length}, names: ${instrumentations.map((i) => i.instrumentationName.replace('@opentelemetry/instrumentation-', '') + '@' + i.instrumentationVersion).join(', ')}`,
);
} catch (error) {
console.error('Failed to add auto-instrumentations:', error);
}
registerInstrumentations({
instrumentations: this.instrumentations,
});

View File

@ -1,5 +1,6 @@
import { Registry } from '@tachybase/utils';
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-grpc';
import { Resource } from '@opentelemetry/resources';
import { BatchSpanProcessor, ConsoleSpanExporter, SpanProcessor } from '@opentelemetry/sdk-trace-base';
import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node';
@ -25,6 +26,13 @@ export class Trace {
this.tracerName = tracerName || 'tachybase-trace';
this.version = version || '';
this.registerProcessor('console', () => new BatchSpanProcessor(new ConsoleSpanExporter()));
// 初始化 OTLP 作为链路追踪 Processor
const newOtlpExporter = new OTLPTraceExporter({
url: process.env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT_GRPC || 'http://localhost:4317',
});
// NOTE: 开发时可替换 BatchSpanProcessor 为 SimpleSpanProcessor 以便监测数据实时上传,区别详见
// https://opentelemetry.io/docs/languages/js/instrumentation/#picking-the-right-span-processor
this.registerProcessor('otlp', () => new BatchSpanProcessor(newOtlpExporter));
}
init(resource: Resource) {
@ -35,6 +43,7 @@ export class Trace {
}
registerProcessor(name: string, processor: GetSpanProcessor) {
console.log('register trace processor:', name);
this.processors.register(name, processor);
}
@ -55,6 +64,13 @@ export class Trace {
const processor = this.getProcessor(name)();
this.provider.addSpanProcessor(processor);
});
// 创建链路追踪的 span自动插桩的 span 才能正常 work我也不知道为什么别删就对了
const tracer = this.getTracer();
const span = tracer.startSpan('load');
span.setAttribute('event', 'load');
span.addEvent('load');
span.end();
}
shutdown() {

View File

@ -1,2 +0,0 @@
/node_modules
/src

View File

@ -1,25 +0,0 @@
# prometheus
English | [中文](./README.zh-CN.md)
## Install
First, edit `.env` to enable the system telemetry and add the prometheus collector
```
TELEMETRY_ENABLED=on
TELEMETRY_METRIC_READER=console,prometheus
TELEMETRY_PROMETHEUS_SERVER=on # If not on, the prometheus server will be registered to prometheus:metrics, note that the standalone prometheus server has NO permission restriction!
TELEMETRY_PROMETHEUS_PORT=9464 # The port of the prometheus server, default to 9464
```
Then enable the plugin
```shell
pnpm pm add @tachybase/plugin-prometheus
pnpm pm enable @tachybase/plugin-prometheus
```
## Usage
TODO

View File

@ -1,25 +0,0 @@
# prometheus
[English](./README.md) | 中文
## 安装激活
首先编辑 `.env`,启用系统的遥测功能并添加采集器 prometheus
```
TELEMETRY_ENABLED=on
TELEMETRY_METRIC_READER=console,prometheus
TELEMETRY_PROMETHEUS_SERVER=on # 如果不为 on则 prometheus 服务器将注册到 prometheus:metrics ,注意 prometheus 的独立服务器*没有*权限限制!
TELEMETRY_PROMETHEUS_PORT=9464 # prometheus 服务器的端口,默认为 9464
```
然后启用插件
```shell
pnpm pm add @tachybase/plugin-prometheus
pnpm pm enable @tachybase/plugin-prometheus
```
## 使用方法
TODO

View File

@ -1,2 +0,0 @@
export * from './dist/client';
export { default } from './dist/client';

View File

@ -1 +0,0 @@
module.exports = require('./dist/client/index.js');

View File

@ -1,26 +0,0 @@
{
"name": "@tachybase/plugin-prometheus",
"displayName": "Prometheus Monitoring",
"version": "0.21.80",
"description": "Prometheus collector for system telemetry",
"keywords": [
"Users & permissions"
],
"license": "Apache-2.0",
"main": "dist/server/index.js",
"dependencies": {
"@opentelemetry/exporter-prometheus": "^0.52.1",
"node-os-utils": "^1.3.7",
"prom-client": "^15.1.3"
},
"peerDependencies": {
"@tachybase/actions": "workspace:*",
"@tachybase/auth": "workspace:*",
"@tachybase/client": "workspace:*",
"@tachybase/database": "workspace:*",
"@tachybase/server": "workspace:*",
"@tachybase/test": "workspace:*"
},
"description.zh-CN": "系统遥测的 Prometheus 采集器",
"displayName.zh-CN": "Prometheus 监控"
}

View File

@ -1,2 +0,0 @@
export * from './dist/server';
export { default } from './dist/server';

View File

@ -1 +0,0 @@
module.exports = require('./dist/server/index.js');

View File

@ -1,21 +0,0 @@
import { Plugin } from '@tachybase/client';
export class PluginPrometheusClient extends Plugin {
async afterAdd() {
// await this.app.pm.add()
}
async beforeLoad() {}
// You can get and modify the app instance here
async load() {
console.log(this.app);
// this.app.addComponents({})
// this.app.addScopes({})
// this.app.addProvider()
// this.app.addProviders()
// this.app.router.add()
}
}
export default PluginPrometheusClient;

View File

@ -1,2 +0,0 @@
export * from './server';
export { default } from './server';

View File

@ -1 +0,0 @@
export { default } from './plugin';

View File

@ -1,16 +0,0 @@
import { hardware } from './hardware';
import { processSelf } from './process';
export default class initMeters {
meter: any;
constructor(meter) {
this.meter = meter;
}
async start() {
const hardwareMeter = new hardware(this.meter);
await hardwareMeter.start();
const processMeter = new processSelf(this.meter);
await processMeter.start();
}
}

View File

@ -1,47 +0,0 @@
import { Plugin } from '@tachybase/server';
import initMeters from './meters';
import { InternalPrometheusExporter, PrometheusExporter } from './prometheus-exporters';
export class PluginPrometheusServer extends Plugin {
async afterAdd() {
this.app.on('beforeLoad', async (app) => {
const startServer = process.env.TELEMETRY_PROMETHEUS_SERVER === 'on';
let reader;
if (!startServer) {
reader = () => new InternalPrometheusExporter(app);
this.log.info('internal prometheus endpoint registered on /api/prometheus:metrics');
} else {
const port: number = Number(process.env.TELEMETRY_PROMETHEUS_PORT) || 9464;
reader = () =>
new PrometheusExporter(
{
port, // optional - default is 9464
},
() => {
this.log.info(`independent prometheus endpoint started on http://localhost:${port}/metrics`);
},
);
}
app.telemetry.metric.registerReader('prometheus', reader);
});
}
async beforeLoad() {}
async load() {
const meter = this.app.telemetry.metric.getMeter();
const meters = new initMeters(meter);
await meters.start();
}
async install() {}
async afterEnable() {}
async afterDisable() {}
async remove() {}
}
export default PluginPrometheusServer;

View File

@ -1,2 +0,0 @@
export * from './internal-prometheus-exporter';
export { PrometheusExporter } from '@opentelemetry/exporter-prometheus';

View File

@ -1,35 +0,0 @@
import { PrometheusExporter, PrometheusSerializer } from '@opentelemetry/exporter-prometheus';
export class InternalPrometheusExporter extends PrometheusExporter {
constructor(app) {
super({
preventServerStart: true,
});
app.resourcer.define({
name: 'prometheus',
actions: {
metrics: async (ctx) => {
ctx.withoutDataWrapping = true;
ctx.set('Content-Type', 'text/plain');
ctx.res.statusCode = 200;
try {
const collectionRes = await this.collect();
const { resourceMetrics, errors } = collectionRes;
if (errors.length) {
ctx.log.error(`metrics collection errors`, {
method: 'InternalPrometheusExporter',
errors,
});
}
ctx.body = new PrometheusSerializer().serialize(resourceMetrics);
} catch (err) {
ctx.body = `# failed to export metrics: ${err}`;
}
},
},
});
}
collect() {
return super.collect();
}
}

File diff suppressed because it is too large Load Diff