feat(telemetry): export traces data in otlp format & matrics data to prometheus (#1400)
Reviewed-on: daoyoucloud/tachybase#1400 Reviewed-by: sealday <zhanglin@daoyoucloud.com> Co-authored-by: TomyJan <TomyJan6@gmail.com> Co-committed-by: TomyJan <TomyJan6@gmail.com>
This commit is contained in:
parent
9261719589
commit
ba9dc69086
11
.env.example
11
.env.example
@ -62,3 +62,14 @@ INIT_ROOT_EMAIL=admin@tachybase.com
|
||||
INIT_ROOT_PASSWORD=!Admin123.
|
||||
INIT_ROOT_NICKNAME=Super Admin
|
||||
INIT_ROOT_USERNAME=tachybase
|
||||
|
||||
################# TELEMETRY #################
|
||||
|
||||
# TELEMETRY_ENABLED=on # 是否启用遥测
|
||||
# TELEMETRY_SERVICE_NAME=tachybase-default-demo # 服务名称,默认为 tachybase-main
|
||||
# OTEL_LOG_LEVEL=debug # OpenTelemetry 日志级别,仅输出到控制台,见 [官方文档](https://open-telemetry.github.io/opentelemetry-js/enums/_opentelemetry_api.DiagLogLevel.html) ,不配置则默认不输出日志
|
||||
# OTEL_METRICS_READER=console,prometheus # 指标读取器,两个均为内置,console 为控制台,prometheus 为 prometheus 服务器
|
||||
# OTEL_PROMETHEUS_SERVER=on # 是否启用 Prometheus 服务,用于创建服务器以供导出指标数据
|
||||
# OTEL_PROMETHEUS_PORT=9464 # Prometheus 服务器端口,默认为 9464,注意此服务器没有鉴权
|
||||
# OTEL_TRACES_PROCESSOR=console,otlp # 链路追踪处理器,两个均为内置,console 为控制台,otlp 为 OTLP 规范的追踪服务器
|
||||
# OTEL_EXPORTER_OTLP_TRACES_ENDPOINT_GRPC=http://localhost:4317 # [OTLP](https://opentelemetry.io/docs/specs/otlp/) 规范的追踪服务器的 gRPC 端点地址
|
||||
|
@ -3,9 +3,9 @@ import { AppTelemetryOptions } from '@tachybase/server';
|
||||
export const telemetry: AppTelemetryOptions = {
|
||||
enabled: process.env.TELEMETRY_ENABLED === 'on',
|
||||
metric: {
|
||||
readerName: process.env.TELEMETRY_METRIC_READER,
|
||||
readerName: process.env.OTEL_METRICS_READER,
|
||||
},
|
||||
trace: {
|
||||
processorName: process.env.TELEMETRY_TRACE_PROCESSOR,
|
||||
processorName: process.env.OTEL_TRACES_PROCESSOR,
|
||||
},
|
||||
};
|
||||
|
@ -1066,8 +1066,13 @@ export class Application<StateT = DefaultState, ContextT = DefaultContext> exten
|
||||
plugins: plugins || [],
|
||||
});
|
||||
|
||||
let serviceName = process.env.TELEMETRY_SERVICE_NAME;
|
||||
if (!serviceName) {
|
||||
this.log.warn('TELEMETRY_SERVICE_NAME is not set, will use default service name, please set it in .env file!');
|
||||
serviceName = `tachybase-${this.name}`;
|
||||
}
|
||||
this._telemetry = new Telemetry({
|
||||
serviceName: `tachybase-${this.name}`,
|
||||
serviceName,
|
||||
version: this.getVersion(),
|
||||
...options.telemetry,
|
||||
});
|
||||
|
@ -7,12 +7,16 @@
|
||||
"types": "./lib/index.d.ts",
|
||||
"dependencies": {
|
||||
"@opentelemetry/api": "^1.7.0",
|
||||
"@opentelemetry/auto-instrumentations-node": "^0.48.0",
|
||||
"@opentelemetry/exporter-prometheus": "^0.52.1",
|
||||
"@opentelemetry/exporter-trace-otlp-grpc": "^0.52.1",
|
||||
"@opentelemetry/instrumentation": "^0.46.0",
|
||||
"@opentelemetry/resources": "^1.19.0",
|
||||
"@opentelemetry/sdk-metrics": "^1.19.0",
|
||||
"@opentelemetry/sdk-trace-base": "^1.19.0",
|
||||
"@opentelemetry/sdk-trace-node": "^1.19.0",
|
||||
"@opentelemetry/semantic-conventions": "^1.19.0",
|
||||
"@tachybase/utils": "workspace:*"
|
||||
"@tachybase/utils": "workspace:*",
|
||||
"node-os-utils": "^1.3.7"
|
||||
}
|
||||
}
|
||||
|
@ -2,13 +2,13 @@ import os from 'os';
|
||||
|
||||
import osUtils from 'node-os-utils';
|
||||
|
||||
export class hardware {
|
||||
export class environmentMeters {
|
||||
meter: any;
|
||||
constructor(meter) {
|
||||
this.meter = meter;
|
||||
}
|
||||
async start() {
|
||||
console.log('hardware meter start');
|
||||
console.log('environment meter start');
|
||||
|
||||
const uptime = this.meter.createObservableGauge('environment.hardware.uptime', {
|
||||
description: 'Uptime in seconds',
|
||||
@ -72,6 +72,6 @@ export class hardware {
|
||||
});
|
||||
}
|
||||
async shutdown() {
|
||||
console.log('hardware meter shutdown');
|
||||
console.log('environment meter shutdown');
|
||||
}
|
||||
}
|
20
packages/core/telemetry/src/meters/index.ts
Normal file
20
packages/core/telemetry/src/meters/index.ts
Normal file
@ -0,0 +1,20 @@
|
||||
import { environmentMeters } from './environment';
|
||||
import { processMeters } from './process';
|
||||
|
||||
export default class initMeters {
|
||||
meter: any;
|
||||
constructor(meter) {
|
||||
this.meter = meter;
|
||||
}
|
||||
async start() {
|
||||
try {
|
||||
const environmentMeter = new environmentMeters(this.meter);
|
||||
await environmentMeter.start();
|
||||
|
||||
const processMeter = new processMeters(this.meter);
|
||||
await processMeter.start();
|
||||
} catch (error) {
|
||||
console.error('Failed to initialize meters:', error);
|
||||
}
|
||||
}
|
||||
}
|
@ -1,4 +1,4 @@
|
||||
export class processSelf {
|
||||
export class processMeters {
|
||||
meter: any;
|
||||
constructor(meter) {
|
||||
this.meter = meter;
|
||||
@ -29,6 +29,6 @@ export class processSelf {
|
||||
});
|
||||
}
|
||||
async shutdown() {
|
||||
console.log('hardware meter shutdown');
|
||||
console.log('process meter shutdown');
|
||||
}
|
||||
}
|
@ -1,6 +1,7 @@
|
||||
import { Registry } from '@tachybase/utils';
|
||||
|
||||
import opentelemetry from '@opentelemetry/api';
|
||||
import { PrometheusExporter } from '@opentelemetry/exporter-prometheus';
|
||||
import { Resource } from '@opentelemetry/resources';
|
||||
import {
|
||||
ConsoleMetricExporter,
|
||||
@ -10,6 +11,8 @@ import {
|
||||
View,
|
||||
} from '@opentelemetry/sdk-metrics';
|
||||
|
||||
import initMeters from './meters';
|
||||
|
||||
export type MetricOptions = {
|
||||
meterName?: string;
|
||||
version?: string;
|
||||
@ -46,6 +49,7 @@ export class Metric {
|
||||
}
|
||||
|
||||
registerReader(name: string, reader: GetMetricReader) {
|
||||
console.log('register metric reader:', name);
|
||||
this.readers.register(name, reader);
|
||||
}
|
||||
|
||||
@ -62,6 +66,31 @@ export class Metric {
|
||||
}
|
||||
|
||||
start() {
|
||||
// 创建 Prometheus 作为指标 Reader
|
||||
try {
|
||||
// 启动 Prometheus 服务器
|
||||
const startServer = process.env.OTEL_PROMETHEUS_SERVER === 'on';
|
||||
if (!startServer) {
|
||||
console.warn('Prometheus server is disabled');
|
||||
} else {
|
||||
const port = Number(process.env.OTEL_PROMETHEUS_PORT) || 9464;
|
||||
const reader = () =>
|
||||
new PrometheusExporter(
|
||||
{
|
||||
port, // optional - default is 9464
|
||||
},
|
||||
() => {
|
||||
console.log(`Prometheus exporter endpoint started on http://localhost:${port}/metrics`);
|
||||
},
|
||||
);
|
||||
// 注册 Prometheus 作为指标 Reader
|
||||
this.registerReader('prometheus', reader);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Failed to initialize Prometheus metrics reader:', error);
|
||||
}
|
||||
|
||||
// 添加指标 Reader
|
||||
let readerName = this.readerName;
|
||||
if (typeof readerName === 'string') {
|
||||
readerName = readerName.split(',');
|
||||
@ -70,6 +99,22 @@ export class Metric {
|
||||
const reader = this.getReader(name)();
|
||||
this.provider.addMetricReader(reader);
|
||||
});
|
||||
|
||||
// 初始化指标 Meters
|
||||
try {
|
||||
const meter = this.getMeter();
|
||||
const meters = new initMeters(meter);
|
||||
meters
|
||||
.start()
|
||||
.then(() => {
|
||||
console.log('Meters initialized');
|
||||
})
|
||||
.catch((error) => {
|
||||
console.error('Failed to initialize meters:', error);
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Failed to initialize meters:', error);
|
||||
}
|
||||
}
|
||||
|
||||
shutdown() {
|
||||
|
@ -1,3 +1,5 @@
|
||||
import { diag, DiagConsoleLogger, DiagLogLevel } from '@opentelemetry/api';
|
||||
import { getNodeAutoInstrumentations } from '@opentelemetry/auto-instrumentations-node';
|
||||
import { InstrumentationOption, registerInstrumentations } from '@opentelemetry/instrumentation';
|
||||
import { Resource } from '@opentelemetry/resources';
|
||||
import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions';
|
||||
@ -29,6 +31,23 @@ export class Telemetry {
|
||||
}
|
||||
|
||||
init() {
|
||||
// 设置 OTel 日志等级
|
||||
const diagLogLevel = process.env.OTEL_LOG_LEVEL;
|
||||
if (diagLogLevel) {
|
||||
diag.setLogger(new DiagConsoleLogger(), DiagLogLevel[diagLogLevel]);
|
||||
}
|
||||
|
||||
// 自动插桩
|
||||
try {
|
||||
const instrumentations = getNodeAutoInstrumentations();
|
||||
this.addInstrumentation(instrumentations);
|
||||
console.log(
|
||||
`auto-instrumentations added, num: ${instrumentations.length}, names: ${instrumentations.map((i) => i.instrumentationName.replace('@opentelemetry/instrumentation-', '') + '@' + i.instrumentationVersion).join(', ')}`,
|
||||
);
|
||||
} catch (error) {
|
||||
console.error('Failed to add auto-instrumentations:', error);
|
||||
}
|
||||
|
||||
registerInstrumentations({
|
||||
instrumentations: this.instrumentations,
|
||||
});
|
||||
|
@ -1,5 +1,6 @@
|
||||
import { Registry } from '@tachybase/utils';
|
||||
|
||||
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-grpc';
|
||||
import { Resource } from '@opentelemetry/resources';
|
||||
import { BatchSpanProcessor, ConsoleSpanExporter, SpanProcessor } from '@opentelemetry/sdk-trace-base';
|
||||
import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node';
|
||||
@ -25,6 +26,13 @@ export class Trace {
|
||||
this.tracerName = tracerName || 'tachybase-trace';
|
||||
this.version = version || '';
|
||||
this.registerProcessor('console', () => new BatchSpanProcessor(new ConsoleSpanExporter()));
|
||||
// 初始化 OTLP 作为链路追踪 Processor
|
||||
const newOtlpExporter = new OTLPTraceExporter({
|
||||
url: process.env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT_GRPC || 'http://localhost:4317',
|
||||
});
|
||||
// NOTE: 开发时可替换 BatchSpanProcessor 为 SimpleSpanProcessor 以便监测数据实时上传,区别详见
|
||||
// https://opentelemetry.io/docs/languages/js/instrumentation/#picking-the-right-span-processor
|
||||
this.registerProcessor('otlp', () => new BatchSpanProcessor(newOtlpExporter));
|
||||
}
|
||||
|
||||
init(resource: Resource) {
|
||||
@ -35,6 +43,7 @@ export class Trace {
|
||||
}
|
||||
|
||||
registerProcessor(name: string, processor: GetSpanProcessor) {
|
||||
console.log('register trace processor:', name);
|
||||
this.processors.register(name, processor);
|
||||
}
|
||||
|
||||
@ -55,6 +64,13 @@ export class Trace {
|
||||
const processor = this.getProcessor(name)();
|
||||
this.provider.addSpanProcessor(processor);
|
||||
});
|
||||
|
||||
// 创建链路追踪的 span,自动插桩的 span 才能正常 work,我也不知道为什么,别删就对了
|
||||
const tracer = this.getTracer();
|
||||
const span = tracer.startSpan('load');
|
||||
span.setAttribute('event', 'load');
|
||||
span.addEvent('load');
|
||||
span.end();
|
||||
}
|
||||
|
||||
shutdown() {
|
||||
|
@ -1,2 +0,0 @@
|
||||
/node_modules
|
||||
/src
|
@ -1,25 +0,0 @@
|
||||
# prometheus
|
||||
|
||||
English | [中文](./README.zh-CN.md)
|
||||
|
||||
## Install
|
||||
|
||||
First, edit `.env` to enable the system telemetry and add the prometheus collector
|
||||
|
||||
```
|
||||
TELEMETRY_ENABLED=on
|
||||
TELEMETRY_METRIC_READER=console,prometheus
|
||||
TELEMETRY_PROMETHEUS_SERVER=on # If not on, the prometheus server will be registered to prometheus:metrics, note that the standalone prometheus server has NO permission restriction!
|
||||
TELEMETRY_PROMETHEUS_PORT=9464 # The port of the prometheus server, default to 9464
|
||||
```
|
||||
|
||||
Then enable the plugin
|
||||
|
||||
```shell
|
||||
pnpm pm add @tachybase/plugin-prometheus
|
||||
pnpm pm enable @tachybase/plugin-prometheus
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
TODO
|
@ -1,25 +0,0 @@
|
||||
# prometheus
|
||||
|
||||
[English](./README.md) | 中文
|
||||
|
||||
## 安装激活
|
||||
|
||||
首先编辑 `.env`,启用系统的遥测功能并添加采集器 prometheus
|
||||
|
||||
```
|
||||
TELEMETRY_ENABLED=on
|
||||
TELEMETRY_METRIC_READER=console,prometheus
|
||||
TELEMETRY_PROMETHEUS_SERVER=on # 如果不为 on,则 prometheus 服务器将注册到 prometheus:metrics ,注意 prometheus 的独立服务器*没有*权限限制!
|
||||
TELEMETRY_PROMETHEUS_PORT=9464 # prometheus 服务器的端口,默认为 9464
|
||||
```
|
||||
|
||||
然后启用插件
|
||||
|
||||
```shell
|
||||
pnpm pm add @tachybase/plugin-prometheus
|
||||
pnpm pm enable @tachybase/plugin-prometheus
|
||||
```
|
||||
|
||||
## 使用方法
|
||||
|
||||
TODO
|
@ -1,2 +0,0 @@
|
||||
export * from './dist/client';
|
||||
export { default } from './dist/client';
|
@ -1 +0,0 @@
|
||||
module.exports = require('./dist/client/index.js');
|
@ -1,26 +0,0 @@
|
||||
{
|
||||
"name": "@tachybase/plugin-prometheus",
|
||||
"displayName": "Prometheus Monitoring",
|
||||
"version": "0.21.80",
|
||||
"description": "Prometheus collector for system telemetry",
|
||||
"keywords": [
|
||||
"Users & permissions"
|
||||
],
|
||||
"license": "Apache-2.0",
|
||||
"main": "dist/server/index.js",
|
||||
"dependencies": {
|
||||
"@opentelemetry/exporter-prometheus": "^0.52.1",
|
||||
"node-os-utils": "^1.3.7",
|
||||
"prom-client": "^15.1.3"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@tachybase/actions": "workspace:*",
|
||||
"@tachybase/auth": "workspace:*",
|
||||
"@tachybase/client": "workspace:*",
|
||||
"@tachybase/database": "workspace:*",
|
||||
"@tachybase/server": "workspace:*",
|
||||
"@tachybase/test": "workspace:*"
|
||||
},
|
||||
"description.zh-CN": "系统遥测的 Prometheus 采集器",
|
||||
"displayName.zh-CN": "Prometheus 监控"
|
||||
}
|
@ -1,2 +0,0 @@
|
||||
export * from './dist/server';
|
||||
export { default } from './dist/server';
|
@ -1 +0,0 @@
|
||||
module.exports = require('./dist/server/index.js');
|
@ -1,21 +0,0 @@
|
||||
import { Plugin } from '@tachybase/client';
|
||||
|
||||
export class PluginPrometheusClient extends Plugin {
|
||||
async afterAdd() {
|
||||
// await this.app.pm.add()
|
||||
}
|
||||
|
||||
async beforeLoad() {}
|
||||
|
||||
// You can get and modify the app instance here
|
||||
async load() {
|
||||
console.log(this.app);
|
||||
// this.app.addComponents({})
|
||||
// this.app.addScopes({})
|
||||
// this.app.addProvider()
|
||||
// this.app.addProviders()
|
||||
// this.app.router.add()
|
||||
}
|
||||
}
|
||||
|
||||
export default PluginPrometheusClient;
|
@ -1,2 +0,0 @@
|
||||
export * from './server';
|
||||
export { default } from './server';
|
@ -1 +0,0 @@
|
||||
export { default } from './plugin';
|
@ -1,16 +0,0 @@
|
||||
import { hardware } from './hardware';
|
||||
import { processSelf } from './process';
|
||||
|
||||
export default class initMeters {
|
||||
meter: any;
|
||||
constructor(meter) {
|
||||
this.meter = meter;
|
||||
}
|
||||
async start() {
|
||||
const hardwareMeter = new hardware(this.meter);
|
||||
await hardwareMeter.start();
|
||||
|
||||
const processMeter = new processSelf(this.meter);
|
||||
await processMeter.start();
|
||||
}
|
||||
}
|
@ -1,47 +0,0 @@
|
||||
import { Plugin } from '@tachybase/server';
|
||||
|
||||
import initMeters from './meters';
|
||||
import { InternalPrometheusExporter, PrometheusExporter } from './prometheus-exporters';
|
||||
|
||||
export class PluginPrometheusServer extends Plugin {
|
||||
async afterAdd() {
|
||||
this.app.on('beforeLoad', async (app) => {
|
||||
const startServer = process.env.TELEMETRY_PROMETHEUS_SERVER === 'on';
|
||||
let reader;
|
||||
if (!startServer) {
|
||||
reader = () => new InternalPrometheusExporter(app);
|
||||
this.log.info('internal prometheus endpoint registered on /api/prometheus:metrics');
|
||||
} else {
|
||||
const port: number = Number(process.env.TELEMETRY_PROMETHEUS_PORT) || 9464;
|
||||
reader = () =>
|
||||
new PrometheusExporter(
|
||||
{
|
||||
port, // optional - default is 9464
|
||||
},
|
||||
() => {
|
||||
this.log.info(`independent prometheus endpoint started on http://localhost:${port}/metrics`);
|
||||
},
|
||||
);
|
||||
}
|
||||
app.telemetry.metric.registerReader('prometheus', reader);
|
||||
});
|
||||
}
|
||||
|
||||
async beforeLoad() {}
|
||||
|
||||
async load() {
|
||||
const meter = this.app.telemetry.metric.getMeter();
|
||||
const meters = new initMeters(meter);
|
||||
await meters.start();
|
||||
}
|
||||
|
||||
async install() {}
|
||||
|
||||
async afterEnable() {}
|
||||
|
||||
async afterDisable() {}
|
||||
|
||||
async remove() {}
|
||||
}
|
||||
|
||||
export default PluginPrometheusServer;
|
@ -1,2 +0,0 @@
|
||||
export * from './internal-prometheus-exporter';
|
||||
export { PrometheusExporter } from '@opentelemetry/exporter-prometheus';
|
@ -1,35 +0,0 @@
|
||||
import { PrometheusExporter, PrometheusSerializer } from '@opentelemetry/exporter-prometheus';
|
||||
|
||||
export class InternalPrometheusExporter extends PrometheusExporter {
|
||||
constructor(app) {
|
||||
super({
|
||||
preventServerStart: true,
|
||||
});
|
||||
app.resourcer.define({
|
||||
name: 'prometheus',
|
||||
actions: {
|
||||
metrics: async (ctx) => {
|
||||
ctx.withoutDataWrapping = true;
|
||||
ctx.set('Content-Type', 'text/plain');
|
||||
ctx.res.statusCode = 200;
|
||||
try {
|
||||
const collectionRes = await this.collect();
|
||||
const { resourceMetrics, errors } = collectionRes;
|
||||
if (errors.length) {
|
||||
ctx.log.error(`metrics collection errors`, {
|
||||
method: 'InternalPrometheusExporter',
|
||||
errors,
|
||||
});
|
||||
}
|
||||
ctx.body = new PrometheusSerializer().serialize(resourceMetrics);
|
||||
} catch (err) {
|
||||
ctx.body = `# failed to export metrics: ${err}`;
|
||||
}
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
collect() {
|
||||
return super.collect();
|
||||
}
|
||||
}
|
1132
pnpm-lock.yaml
1132
pnpm-lock.yaml
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue
Block a user