Apache Druid vs Apache Pinot

Audits failed Kafka indexing tasks and datasource segment distribution to diagnose ingestion worker starvation.

-- Query Druid system tables to audit failed indexing tasks
SELECT
  task_id,
  type,
  datasource,
  status,
  error_msg,
  created_time,
  duration
FROM sys.tasks
WHERE status = 'FAILED'
  AND created_time >= CURRENT_TIMESTAMP - INTERVAL '24' HOUR
ORDER BY created_time DESC
LIMIT 10;

-- Audit active segments and total disk footprint by datasource
SELECT
  datasource,
  COUNT(*) AS total_segments,
  ROUND(SUM(size) / 1024 / 1024 / 1024, 2) AS size_gb,
  SUM(num_rows) AS total_rows
FROM sys.segments
WHERE is_active = 1
GROUP BY datasource
ORDER BY size_gb DESC;

Audits table debug metrics, segment assignment counts, and server status directly via the Pinot Controller REST API.

# Inspect cluster table list and server status
curl -s "http://pinot-controller:9000/tables" | jq .

# Inspect table debug metrics and segment partition assignments
curl -s "http://pinot-controller:9000/tables/realtimeSales_REALTIME/debug" | jq '{
  tableName: .tableName,
  segmentCount: .segmentCount,
  unassignedSegments: .unassignedSegments,
  serverStatus: .serverStatus
}'

# Check query latency stats and exceptions from Pinot Broker logs
tail -n 100 /var/log/pinot/pinot-broker.log | grep -E "Exception|QueryExecutionError|timeMs"