Health, metrics and dead letters
Probes for your orchestrator, metrics for Prometheus, and the dead-letter queue for messages that failed every attempt.
Liveness
GET/live
Auth: None.
Responses
200 OK Always, while the process runs.
{
"status": "ok"
}
Common errors (500)
{
"error": "internal_error"
}
Example
curl http://localhost:3000/live
Readiness
GET/ready
Auth: None.
Use it as the readiness or deploy healthcheck.
Responses
{
"status": "ready"
}
{
"status": "unready"
}
Common errors (500)
{
"error": "internal_error"
}
Example
curl http://localhost:3000/ready
Health
GET/health
Auth: None.
Dependencies plus each worker's heartbeat, cached for one second. Heartbeats expire after 15 seconds, so a worker in another deployment, or a dead one, shows as unknown.
Responses
{
"service": "api",
"status": "ok",
"redis": true,
"database": true,
"workers": {
"enricher": {
"service": "enricher",
"redis": true,
"state": "running",
"processedCount": 4812,
"errorCount": 3,
"lastProcessedAt": "2026-11-17T10:00:01.204Z",
"lastErrorAt": "2026-11-17T09:41:12.003Z",
"pendingCount": 0,
"updatedAt": "2026-11-17T10:00:05.000Z"
},
"engine": {
"service": "engine",
"redis": true,
"state": "running",
"processedCount": 4812,
"errorCount": 3,
"lastProcessedAt": "2026-11-17T10:00:01.204Z",
"lastErrorAt": "2026-11-17T09:41:12.003Z",
"pendingCount": 0,
"updatedAt": "2026-11-17T10:00:05.000Z"
},
"scheduler": {
"service": "scheduler",
"redis": true,
"state": "running",
"processedCount": 4812,
"errorCount": 3,
"lastProcessedAt": "2026-11-17T10:00:01.204Z",
"lastErrorAt": "2026-11-17T09:41:12.003Z",
"pendingCount": 0,
"updatedAt": "2026-11-17T10:00:05.000Z"
},
"delivery": {
"service": "delivery",
"redis": true,
"state": "running",
"processedCount": 4812,
"errorCount": 3,
"lastProcessedAt": "2026-11-17T10:00:01.204Z",
"lastErrorAt": "2026-11-17T09:41:12.003Z",
"pendingCount": 0,
"updatedAt": "2026-11-17T10:00:05.000Z"
},
"ai": {
"status": "unknown",
"message": "No report received from worker"
},
"workflow": {
"service": "workflow",
"redis": true,
"state": "running",
"processedCount": 4812,
"errorCount": 3,
"lastProcessedAt": "2026-11-17T10:00:01.204Z",
"lastErrorAt": "2026-11-17T09:41:12.003Z",
"pendingCount": 0,
"updatedAt": "2026-11-17T10:00:05.000Z"
},
"events": {
"service": "events",
"redis": true,
"state": "running",
"processedCount": 4812,
"errorCount": 3,
"lastProcessedAt": "2026-11-17T10:00:01.204Z",
"lastErrorAt": "2026-11-17T09:41:12.003Z",
"pendingCount": 0,
"updatedAt": "2026-11-17T10:00:05.000Z"
}
}
}
{
"service": "api",
"status": "degraded",
"redis": true,
"database": false,
"workers": {
"enricher": {
"service": "enricher",
"redis": true,
"state": "running",
"processedCount": 4812,
"errorCount": 3,
"lastProcessedAt": "2026-11-17T10:00:01.204Z",
"lastErrorAt": "2026-11-17T09:41:12.003Z",
"pendingCount": 0,
"updatedAt": "2026-11-17T10:00:05.000Z"
},
"engine": {
"service": "engine",
"redis": true,
"state": "running",
"processedCount": 4812,
"errorCount": 3,
"lastProcessedAt": "2026-11-17T10:00:01.204Z",
"lastErrorAt": "2026-11-17T09:41:12.003Z",
"pendingCount": 0,
"updatedAt": "2026-11-17T10:00:05.000Z"
},
"scheduler": {
"service": "scheduler",
"redis": true,
"state": "running",
"processedCount": 4812,
"errorCount": 3,
"lastProcessedAt": "2026-11-17T10:00:01.204Z",
"lastErrorAt": "2026-11-17T09:41:12.003Z",
"pendingCount": 0,
"updatedAt": "2026-11-17T10:00:05.000Z"
},
"delivery": {
"service": "delivery",
"redis": true,
"state": "running",
"processedCount": 4812,
"errorCount": 3,
"lastProcessedAt": "2026-11-17T10:00:01.204Z",
"lastErrorAt": "2026-11-17T09:41:12.003Z",
"pendingCount": 0,
"updatedAt": "2026-11-17T10:00:05.000Z"
},
"ai": {
"status": "unknown",
"message": "No report received from worker"
},
"workflow": {
"service": "workflow",
"redis": true,
"state": "running",
"processedCount": 4812,
"errorCount": 3,
"lastProcessedAt": "2026-11-17T10:00:01.204Z",
"lastErrorAt": "2026-11-17T09:41:12.003Z",
"pendingCount": 0,
"updatedAt": "2026-11-17T10:00:05.000Z"
},
"events": {
"service": "events",
"redis": true,
"state": "running",
"processedCount": 4812,
"errorCount": 3,
"lastProcessedAt": "2026-11-17T10:00:01.204Z",
"lastErrorAt": "2026-11-17T09:41:12.003Z",
"pendingCount": 0,
"updatedAt": "2026-11-17T10:00:05.000Z"
}
}
}
{
"service": "api",
"status": "degraded",
"redis": false,
"database": true,
"workers": {}
}
Common errors (500)
{
"error": "internal_error"
}
Example
curl http://localhost:3000/health
Prometheus metrics
GET/metrics
Auth: None from loopback. Otherwise the admin key or a session.
Metric names are listed in Operations.
Responses
# HELP notifkit_delivery_success_total Total successful deliveries
# TYPE notifkit_delivery_success_total counter
notifkit_delivery_success_total{channel="email"} 4790
{
"error": "unauthorized",
"message": "Metrics endpoint requires admin credentials"
}
Example
curl http://localhost:3000/metrics -H "Authorization: Bearer $ADMIN_API_KEY"
System health
GET/v1/system/health
Auth: Any project key, the admin key or a session. No x-project-id needed.
Like /health, with latencies in milliseconds. Always 200: read status.
Responses
{
"status": "healthy",
"redis": {
"ok": true,
"latencyMs": 1
},
"db": {
"ok": true,
"latencyMs": 3
},
"workers": {
"enricher": {
"service": "enricher",
"redis": true,
"state": "running",
"processedCount": 4812,
"errorCount": 3,
"lastProcessedAt": "2026-11-17T10:00:01.204Z",
"lastErrorAt": "2026-11-17T09:41:12.003Z",
"pendingCount": 0,
"updatedAt": "2026-11-17T10:00:05.000Z"
},
"engine": {
"service": "engine",
"redis": true,
"state": "running",
"processedCount": 4812,
"errorCount": 3,
"lastProcessedAt": "2026-11-17T10:00:01.204Z",
"lastErrorAt": "2026-11-17T09:41:12.003Z",
"pendingCount": 0,
"updatedAt": "2026-11-17T10:00:05.000Z"
},
"scheduler": {
"service": "scheduler",
"redis": true,
"state": "running",
"processedCount": 4812,
"errorCount": 3,
"lastProcessedAt": "2026-11-17T10:00:01.204Z",
"lastErrorAt": "2026-11-17T09:41:12.003Z",
"pendingCount": 0,
"updatedAt": "2026-11-17T10:00:05.000Z"
},
"delivery": {
"service": "delivery",
"redis": true,
"state": "running",
"processedCount": 4812,
"errorCount": 3,
"lastProcessedAt": "2026-11-17T10:00:01.204Z",
"lastErrorAt": "2026-11-17T09:41:12.003Z",
"pendingCount": 0,
"updatedAt": "2026-11-17T10:00:05.000Z"
},
"ai": {
"status": "unknown",
"message": "No heartbeat"
},
"workflow": {
"service": "workflow",
"redis": true,
"state": "running",
"processedCount": 4812,
"errorCount": 3,
"lastProcessedAt": "2026-11-17T10:00:01.204Z",
"lastErrorAt": "2026-11-17T09:41:12.003Z",
"pendingCount": 0,
"updatedAt": "2026-11-17T10:00:05.000Z"
},
"events": {
"service": "events",
"redis": true,
"state": "running",
"processedCount": 4812,
"errorCount": 3,
"lastProcessedAt": "2026-11-17T10:00:01.204Z",
"lastErrorAt": "2026-11-17T09:41:12.003Z",
"pendingCount": 0,
"updatedAt": "2026-11-17T10:00:05.000Z"
}
}
}
{
"status": "degraded",
"redis": {
"ok": true,
"latencyMs": 1
},
"db": {
"ok": false,
"latencyMs": 0
},
"workers": {
"enricher": {
"service": "enricher",
"redis": true,
"state": "running",
"processedCount": 4812,
"errorCount": 3,
"lastProcessedAt": "2026-11-17T10:00:01.204Z",
"lastErrorAt": "2026-11-17T09:41:12.003Z",
"pendingCount": 0,
"updatedAt": "2026-11-17T10:00:05.000Z"
},
"engine": {
"service": "engine",
"redis": true,
"state": "running",
"processedCount": 4812,
"errorCount": 3,
"lastProcessedAt": "2026-11-17T10:00:01.204Z",
"lastErrorAt": "2026-11-17T09:41:12.003Z",
"pendingCount": 0,
"updatedAt": "2026-11-17T10:00:05.000Z"
},
"scheduler": {
"service": "scheduler",
"redis": true,
"state": "running",
"processedCount": 4812,
"errorCount": 3,
"lastProcessedAt": "2026-11-17T10:00:01.204Z",
"lastErrorAt": "2026-11-17T09:41:12.003Z",
"pendingCount": 0,
"updatedAt": "2026-11-17T10:00:05.000Z"
},
"delivery": {
"service": "delivery",
"redis": true,
"state": "running",
"processedCount": 4812,
"errorCount": 3,
"lastProcessedAt": "2026-11-17T10:00:01.204Z",
"lastErrorAt": "2026-11-17T09:41:12.003Z",
"pendingCount": 0,
"updatedAt": "2026-11-17T10:00:05.000Z"
},
"ai": {
"status": "unknown",
"message": "No heartbeat"
},
"workflow": {
"service": "workflow",
"redis": true,
"state": "running",
"processedCount": 4812,
"errorCount": 3,
"lastProcessedAt": "2026-11-17T10:00:01.204Z",
"lastErrorAt": "2026-11-17T09:41:12.003Z",
"pendingCount": 0,
"updatedAt": "2026-11-17T10:00:05.000Z"
},
"events": {
"service": "events",
"redis": true,
"state": "running",
"processedCount": 4812,
"errorCount": 3,
"lastProcessedAt": "2026-11-17T10:00:01.204Z",
"lastErrorAt": "2026-11-17T09:41:12.003Z",
"pendingCount": 0,
"updatedAt": "2026-11-17T10:00:05.000Z"
}
}
}
{
"status": "degraded",
"redis": {
"ok": false,
"latencyMs": 0
},
"db": {
"ok": true,
"latencyMs": 3
},
"workers": {}
}
Common errors (401, 429, 500)
{
"error": "unauthorized",
"message": "Invalid or missing API key"
}
{
"error": "too_many_requests",
"message": "Project rate limit exceeded (max 600 req/min)"
}
{
"error": "internal_error"
}
Example
curl http://localhost:3000/v1/system/health -H "Authorization: Bearer $NK_KEY"
const health = await notifkit.getSystemHealth();
System metrics
GET/v1/system/metrics
Auth: Any project key, the admin key or a session. No x-project-id needed.
Backlog per stream (unread plus unacknowledged) and delivery totals, counted in tasks. Stream depths are shared infrastructure, so only the admin key and sessions see them. successRate is 100 when nothing has been sent.
Responses
{
"streams": {
"INBOUND_CRITICAL": 0,
"INBOUND_NORMAL": 0,
"INBOUND_LOW": 0,
"ENRICHED_NORMAL": 0,
"OUTBOUND_CRITICAL": 0,
"OUTBOUND_NORMAL": 12,
"OUTBOUND_LOW": 0,
"WORKFLOW_INBOUND": 0,
"EVENTS_INBOUND": 3,
"DEAD_LETTER": 2
},
"deliveryStats": {
"total": 4812,
"delivered": 4790,
"failed": 22,
"successRate": 99.54
}
}
{
"streams": {
"INBOUND_CRITICAL": 0,
"INBOUND_NORMAL": 0,
"INBOUND_LOW": 0,
"ENRICHED_NORMAL": 0,
"OUTBOUND_CRITICAL": 0,
"OUTBOUND_NORMAL": 12,
"OUTBOUND_LOW": 0,
"WORKFLOW_INBOUND": 0,
"EVENTS_INBOUND": 3,
"DEAD_LETTER": 2
},
"deliveryStats": {
"total": 9410,
"delivered": 9366,
"failed": 2,
"successRate": 99.53
}
}
{
"streams": {},
"deliveryStats": {
"total": 4812,
"delivered": 4790,
"failed": 22,
"successRate": 99.54
}
}
Common errors (401, 429, 500)
{
"error": "unauthorized",
"message": "Invalid or missing API key"
}
{
"error": "too_many_requests",
"message": "Project rate limit exceeded (max 600 req/min)"
}
{
"error": "internal_error"
}
Example
curl http://localhost:3000/v1/system/metrics -H "Authorization: Bearer $ADMIN_API_KEY"
const { streams, deliveryStats } = await notifkit.getSystemMetrics();
Dead-letter queue
Messages that failed every attempt, or hit a non-retryable error. Nothing is retried automatically. The queue is shared across projects: a project key sees only its own entries, and the admin key without x-project-id sees all of them.
List dead letters
GET/v1/dlq
Auth: Any project key, the admin key or a session. No x-project-id needed.
Up to 50 entries, newest first, from the newest 100 in the stream. payload is the message that failed, so its shape depends on eventType.
Responses
{
"messages": [
{
"id": "1731837600000-0",
"eventType": "notification.dispatched",
"payload": {
"projectId": "b2a4c1d0-5e6f-4a7b-8c9d-0e1f2a3b4c5d",
"taskId": "7e0d3f2a-1b4c-4d5e-8f60-7a8b9c0d1e2f:41a9e6b2-3c4d-4e5f-9a6b-7c8d9e0f1a2b",
"enrichedEventId": "3a9b8c7d-6e5f-4a3b-9c2d-1e0f9a8b7c6d",
"recipientId": "usr_123",
"channel": "email",
"priority": "normal",
"templateId": "order-confirmed",
"templateVariables": {
"orderId": "99214"
},
"recipient": {
"id": "usr_123",
"email": "[email protected]",
"locale": "en",
"timezone": "America/New_York",
"preferences": {
"optedOut": false,
"channels": [],
"quietHours": [
{
"start": "22:00",
"end": "08:00"
}
]
}
},
"destination": "[email protected]",
"deliveryOptions": {
"maxAttempts": 3,
"timeoutMs": 10000
},
"deliveryAttemptCount": 2
},
"error": "Transport timeout after 10000ms",
"timestamp": "2026-11-17T10:00:00.000Z",
"originalStream": "notifkit:stream:outbound:normal"
}
]
}
{
"messages": []
}
Common errors (401, 429, 500)
{
"error": "unauthorized",
"message": "Invalid or missing API key"
}
{
"error": "too_many_requests",
"message": "Project rate limit exceeded (max 600 req/min)"
}
{
"error": "internal_error"
}
Example
curl http://localhost:3000/v1/dlq -H "Authorization: Bearer $NK_KEY"
const { messages } = await notifkit.getDLQMessages();
Replay a dead letter
POST/v1/dlq/replay
Auth: Any project key, the admin key or a session. No x-project-id needed.
Puts the message back on the stream it failed on and removes it from the queue. Fix the cause first, or it lands right back.
Body
| Field | Type | Required | Description |
|---|---|---|---|
id | string | yes | Entry id from the list. |
Responses
{
"success": true,
"replayedId": "1731837600000-0",
"stream": "notifkit:stream:outbound:normal"
}
{
"error": "missing_message_id"
}
{
"error": "dlq_message_not_found"
}
{
"error": "replay_failed"
}
Common errors (401, 403, 429, 400, 413, 500)
{
"error": "unauthorized",
"message": "Invalid or missing API key"
}
{
"error": "forbidden",
"message": "API key is read-only"
}
{
"error": "too_many_requests",
"message": "Project rate limit exceeded (max 600 req/min)"
}
{
"error": "malformed_json",
"message": "malformed_json"
}
{
"error": "payload_too_large",
"message": "payload_too_large"
}
{
"error": "internal_error"
}
Example
curl -X POST http://localhost:3000/v1/dlq/replay \
-H "Authorization: Bearer $NK_KEY" \
-H "Content-Type: application/json" \
-d '{ "id": "1731837600000-0" }'
await notifkit.replayDLQMessage("1731837600000-0");
Delete a dead letter
DELETE/v1/dlq/{id}
Auth: Any project key, the admin key or a session. No x-project-id needed.
Path parameters
| Name | Type | Description |
|---|---|---|
id | string | Entry id. |
Responses
{
"success": true
}
{
"error": "dlq_message_not_found"
}
{
"error": "delete_failed"
}
Common errors (401, 403, 429, 500)
{
"error": "unauthorized",
"message": "Invalid or missing API key"
}
{
"error": "forbidden",
"message": "API key is read-only"
}
{
"error": "too_many_requests",
"message": "Project rate limit exceeded (max 600 req/min)"
}
{
"error": "internal_error"
}
Example
curl -X DELETE http://localhost:3000/v1/dlq/1731837600000-0 -H "Authorization: Bearer $NK_KEY"
await notifkit.deleteDLQMessage("1731837600000-0");