Elastic

Elastic ingestion source for Streamfold.

The Elastic source emulates the Elastic bulk API to support clients that can talk to Elastic endpoints, like the Beats collection of data shippers. Ingested documents are converted to events internally and can be transformed or routed to any destination.

Overview

An Elastic source allows you to accept traffic from an Elastic compatible data shipper. Traffic will be accepted at /streamfold/elastic on your ingress endpoint. This source has been tested with the OSS Filebeat data shipper, we'll expand testing and support in the future.

Configuration

Configuration is supplied below for the Filebeat log shipper using the elasticsearch output. The Streamfold Elastic source does not support index lifecycle management so make sure you disable it with setup.ilm.enabled: false.

Replace the correct ingress endpoint and <sf-ingress-token> available in the Source configuration instructions in the application. Learn more

# We don't support index lifecycle management
setup.ilm.enabled: false

output.elasticsearch:
  # Array of hosts to connect to.
  hosts: ["https://ingress.streamfold.com/streamfold/elastic"]

  # Protocol - either `http` (default) or `https`.
  protocol: "https"

  # Authentication credentials - either API key or username/password.
  username: "streamfold"
  password: "<sf-ingress-token>"

  # Support an older version of the API than the client
  allow_older_versions: true

Event example

The following is an example of a payload from the Filebeat log shipper when ingested to Streamfold. The @type of an elastic event is elastic_log.

{
  "container": {
    "id": "3cb46acfb971815be0b8d7dd6bef3be76f8feec4ecbf2c6953d3e1b1e2fcc676",
    "runtime": "containerd",
    "image": {
      "name": "ghcr.io/open-telemetry/demo:1.4.0-currencyservice"
    }
  },
  "log": {
    "offset": 758225,
    "file": {
      "path": "/var/log/containers/otel-sf-currencyservice-5fbbfb5469-7k2jt_default_currencyservice-3cb46acfb971815be0b8d7dd6bef3be76f8feec4ecbf2c6953d3e1b1e2fcc676.log"
    }
  },
  "stream": "stdout",
  "input": {
    "type": "container"
  },
  "agent": {
    "name": "ip-172-31-27-175",
    "type": "filebeat",
    "version": "8.9.1",
    "ephemeral_id": "fea699f3-c0d3-4e45-8258-d2282bb30157",
    "id": "86eb39c8-a7c7-47b8-bfc2-e7629ac3b09e"
  },
  "host": {
    "hostname": "ip-172-31-27-175",
    "name": "ip-172-31-27-175",
    "architecture": "x86_64",
    "os": {
      "codename": "focal",
      "type": "linux",
      "platform": "ubuntu",
      "version": "20.04.6 LTS (Focal Fossa)",
      "family": "debian",
      "name": "Ubuntu",
      "kernel": "5.19.0-1028-aws"
    },
    "containerized": false,
    "ip": [
      "172.31.27.175",
      "fe80::877:3eff:fef3:1fe5"
    ],
    "mac": [
      "0A-77-3E-F3-1F-E5"
    ]
  },
  "ecs": {
    "version": "8.0.0"
  },
  "kubernetes": {
    "pod": {
      "uid": "2eca8238-56f2-47ce-9c18-dd4b35e2d009",
      "ip": "10.42.1.224",
      "name": "otel-sf-currencyservice-5fbbfb5469-7k2jt"
    },
    "namespace_labels": {
      "kubernetes_io/metadata_name": "default"
    },
    "container": {
      "name": "currencyservice"
    },
    "replicaset": {
      "name": "otel-sf-currencyservice-5fbbfb5469"
    },
    "node": {
      "hostname": "ip-172-31-27-175",
      "name": "ip-172-31-27-175",
      "uid": "45ba3db1-275a-4086-8f0a-7d96da1c1dbe",
      "labels": {
        "kubernetes_io/hostname": "ip-172-31-27-175",
        "node_kubernetes_io/instance-type": "k3s",
        "node-role_kubernetes_io/control-plane": "true"
      }
    },
    "namespace_uid": "5ab7038d-cc9d-4548-8a8e-8f1ac77e6ffb",
    "labels": {
      "app_kubernetes_io/component": "currencyservice",
      "app_kubernetes_io/name": "otel-sf-currencyservice"
    },
    "deployment": {
      "name": "otel-sf-currencyservice"
    },
    "namespace": "default"
  },
  "message": "[Error] [OTLP METRIC GRPC Exporter] Export() failed: DNS resolution failed: Domain name not found",
  "@timestamp": "2023-09-06T17:12:08.919Z",
  "cloud": {
    "availability_zone": "us-east-1c",
    "service": {
      "name": "Nova"
    },
    "provider": "openstack",
    "instance": {
      "id": "i-ffff3e272f6fa8fff",
      "name": "ip-172-31-27-175.ec2.internal"
    },
    "machine": {
      "type": "t3.large"
    }
  }
}