Log Shipper Configuration

$ curl -fsSL https://raw.githubusercontent.com/fluent/fluent-bit/master/install.sh | sh
$ /opt/fluent-bit/bin/fluent-bit --version

$ sudo vi /etc/fluent-bit/kaia-logs.conf

# /etc/fluent-bit/kaia-logs.conf
#
# Ships this node's kcnd log to VictoriaLogs via Fluent Bit rather than
# telegraf: telegraf's own InfluxDB output would exit entirely if a second,
# log-shipping output (e.g. Elasticsearch) failed its connection handshake,
# which would silently stop this node's METRICS too. A separate process
# can't do that — whatever happens to log shipping, telegraf and the chain
# process are untouched.

[SERVICE]
    Flush                     5
    Log_Level                 info
    # Node-side spill buffer. Only holds data while the upstream is
    # unreachable, and empties itself once delivery resumes. A queue, not a
    # log store.
    storage.path              /var/lib/fluent-bit/chunks
    # Write chunks through to disk rather than relying on the page cache —
    # the setting that makes a hard kill survivable.
    storage.sync              normal
    storage.checksum          on
    storage.max_chunks_up     128

[INPUT]
    Name                      tail
    Path                      /kaia/kcnd/logs/kcnd.out
    # Read position, so an agent restart neither re-ships nor skips. Must
    # live on persistent storage, not /tmp.
    DB                        /var/lib/fluent-bit/tail.db
    DB.sync                   normal
    storage.type              filesystem
    # Start at the end of the file on first install — shipping the node's
    # entire existing log history on day one would be hours of backlog for
    # no benefit.
    Read_from_Head            false
    # A Kaia log line is ~211 bytes; anything past 32k is not a log line.
    Skip_Long_Lines           on
    Buffer_Chunk_Size         32k
    Buffer_Max_Size           64k
    Tag                       kaia

[FILTER]
    Name                      record_modifier
    Match                     kaia
    # These three become the VictoriaLogs stream, which is what the store
    # partitions and compresses by. Identity only — nothing with open
    # cardinality, and deliberately not the file path (which drifts across
    # the fleet) so each node is its own stream regardless of path shape.
    Record                    instance <hostname>
    Record                    nodetype cn
    Record                    network  mainnet

[OUTPUT]
    Name                      http
    Match                     kaia
    host                      logs.node.kaia.io
    port                      45560
    # No TLS — a deliberate carry-over, same as the metrics path beside it
    # (metrics.node.kaia.io:45560): plain HTTP reachable only from an IP
    # allowlist. Node logs are block numbers/hashes/peer counts, not
    # secrets, but raising this whole ingest surface to TLS is tracked as a
    # separate, not-yet-done item.
    uri                       /insert/jsonline?_msg_field=log&_time_field=date&_stream_fields=instance,nodetype,network
    format                    json_lines
    json_date_format          iso8601
    json_date_key             date
    compress                  gzip
    # Retry forever — the filesystem buffer above holds the backlog, and
    # giving up would turn a cluster restart into permanent loss.
    Retry_Limit               false
    # Cap what this node may hold in the local spill buffer.
    storage.total_limit_size  1G

$ sudo cp /etc/fluent-bit/fluent-bit.conf /etc/fluent-bit/fluent-bit.conf.orig
$ echo "@INCLUDE /etc/fluent-bit/kaia-logs.conf" | sudo tee /etc/fluent-bit/fluent-bit.conf

$ sudo mkdir -p /var/lib/fluent-bit
$ sudo /opt/fluent-bit/bin/fluent-bit --dry-run -c /etc/fluent-bit/fluent-bit.conf