filebeat.yml

################## Enterprise Search Filebeat Configuration Example #####################

# This file is an example configuration file for ingesting Enterprise Search logs into
# Elasticsearch highlighting only the most common options.
#
# Logs will be ingested into filebeat-{filebeat-version}
#
# You can find the full configuration reference here:
# https://www.elastic.co/guide/en/beats/filebeat/index.html

# For more available modules and options, please see the filebeat.reference.yml sample
# configuration file in Filebeat install directory.

# ---------------------------- Elasticsearch Output ----------------------------
output.elasticsearch:
  hosts: ["<elasticsearch_host>"]
  username: "<elasticsearch_username>"
  password: "<elasticsearch_password>"
  # Index name for Enterprise Search logs
  index: "ent-search-logs-8"
  bulk_max_size: 50
  # Set to "none" if your Elasticsearch instance is not secured.
  ssl.verification_mode: full
  ssl.enabled: true
  ssl.certificate: "<path_to_elasticsearch_cert_pem>"
  ssl.key: "<path_to_elasticsearch_cert_key>"

setup.template:
  overwrite: false
  json:
    enabled: true
    path: "filebeat-template-8.json"
    name: "ent-search-logs-8"
  name: "ent-search-logs-8"
  pattern: "ent-search-logs-8*"

setup.ilm:
  policy_name: "ent-search-logs"

processors:
  - drop_event.when.has_fields:
      - labels.skip_ingest

# -------------------------------- Kibana Setup --------------------------------
setup.kibana:
  host: "<kibana_host>" # e.g. https://localhost:5601
  # Set to "none" if your Kibana instance is not secured.
  ssl.verification_mode: full
  ssl.enabled: true
  ssl.certificate: "<path_to_elasticsearch_cert_pem>"
  ssl.key: "<path_to_elasticsearch_cert_key>"

# ============================= Filebeat Logging ===============================
logging.level: info
logging.to_files: true
logging.json: true
logging.files:
  path: <filebeat_log_dir>
  name: filebeat.log
  keepfiles: 3

# ============================== Filebeat inputs ===============================
filebeat.inputs:
  # Enterprise Search main log
  - type: filestream
    id: enterprise_search_logs
    paths:
      - "<enterprise-search-dir>/log/app-server.log*"
    parsers:
      - multiline:
          type: pattern
          pattern: '^\['
          negate: true
          match: after
    fields:
      event.dataset: enterprise_search.server
      # The following fields are set as an example for identifying the deployment,
      #  Enterprise Search version, and specific nodes. Any fields specified in this
      #  section will be added to the log events
      service.name: '<deployment_name>'
      service.id: '<deployment_id>'
      service.version: '<instance_version>'
      service.node.name: '<instance_id>'
    fields_under_root: true

    # Workplace Search connectors log
  - type: filestream
    id: connector_logs
    paths:
      - "<enterprise-search-dir>/log/connectors.log*"
    parsers:
      - multiline:
          type: pattern
          pattern: '^\['
          negate: true
          match: after
    fields:
      event.dataset: enterprise_search.connectors
      service.name: '<deployment_name>'
      service.id: '<deployment_id>'
      service.version: '<instance_version>'
      service.node.name: '<instance_id>'
    fields_under_root: true

    # Crawler log
  - type: filestream
    id: crawler_logs
    paths:
      - "<enterprise-search-dir>/log/crawler.log*"
    parsers:
      - multiline:
          type: pattern
          pattern: '^\['
          negate: true
          match: after
    fields:
      event.dataset: enterprise_search.crawler
      service.name: '<deployment_name>'
      service.id: '<deployment_id>'
      service.version: '<instance_version>'
      service.node.name: '<instance_id>'
    fields_under_root: true

    # API and Analytics log
  - type: filestream
    id: api_analytics_logs
    paths:
      - "<enterprise-search-dir>/log/filebeat.log*"
    parsers:
      - ndjson:
          keys_under_root: true
          overwrite_keys: true
          add_error_key: true
    processors:
      - drop_fields:
          fields: ["data_stream"]
          ignore_missing: false
    fields:
      service.name: '<deployment_name>'
      service.id: '<deployment_id>'
      service.version: '<instance_version>'
      service.node.name: '<instance_id>'
    fields_under_root: true


    # HTTP Requests log
  - type: filestream
    id: http_requests_logs
    paths:
      - "<enterprise-search-dir>/log/system.log*"
    parsers:
      - ndjson:
          keys_under_root: true
          overwrite_keys: true
          add_error_key: true
    fields:
      event.dataset: enterprise_search.http_requests
      service.name: '<deployment_name>'
      service.id: '<deployment_id>'
      service.version: '<instance_version>'
      service.node.name: '<instance_id>'
    fields_under_root: true
    processors:
      - timestamp:
          field: timestamp
          layouts:
            - '2006-01-02T15:04:05-07:00'
      - drop_fields:
          fields: [timestamp]
      - rename:
          ignore_missing: true
          fields:
            - from: method
              to: http.request.method
            - from: http_version
              to: http.version
            - from: status
              to: http.response.status_code
            - from: request_body_size
              to: http.request.body.bytes
            - from: response_body_size
              to: http.response.body.bytes
            - from: path
              to: url.path
            - from: host
              to: url.original