Basic Cluster

The Basic Cluster (medium) deployment is suitable for moderate ingest rates where redundancy is a requirement. It also allows for minimal to no downtime for most maintenance tasks.

Sizing Parameter	Value
Licensed Units	up to 8
Recommended Max. Ingest Rate	24000 flows/sec
Retention at Max. Rate	10 days
Shards	3
Replicas	1

Elasticsearch nodes, Kibana and the ElastiFlow collectors are all installed on dedicated systems.

Application	CPU Cores	Memory	Storage
Kibana	4	16 GB	128 GB

Application	CPU Cores	Memory	SSD Storage
Elasticsearch (master/data)	12-16	64 GB	2 x 4 TB (6.8 TB)
Elasticsearch (master/data)	12-16	64 GB	2 x 4 TB (6.8 TB)
Elasticsearch (master/data)	12-16	64 GB	2 x 4 TB (6.8 TB)

Application	CPU Cores	Memory	Storage
Flow Collector	8	16 GB	128 GB

Docker Compose Configurations

Kibana

version: '3'
services:
  kibana:
    image: docker.elastic.co/kibana/kibana:7.13.1
    restart: unless-stopped
    hostname: KIB_NODE_NAME
    network_mode: bridge
    ports:
      # HTTP/REST
      - 5601:5601/tcp
    environment:
      TELEMETRY_OPTIN: 'false'
      TELEMETRY_ENABLED: 'false'
      NEWSFEED_ENABLED: 'false'

      SERVER_NAME: 'KIB_NODE_NAME'
      SERVER_HOST: '0.0.0.0'
      SERVER_PORT: 5601
      SERVER_MAXPAYLOADBYTES: 8388608

      ELASTICSEARCH_HOSTS: '["https://192.0.2.11:9200","https://192.0.2.12:9200","https://192.0.2.13:9200"]'
      ELASTICSEARCH_USERNAME: 'kibana_system'
      ELASTICSEARCH_PASSWORD: 'CHANGEME'
      ELASTICSEARCH_REQUESTTIMEOUT: 132000
      ELASTICSEARCH_SHARDTIMEOUT: 120000

      #ELASTICSEARCH_SSL_CERTIFICATE: /etc/kibana/certs/node/node.crt
      #ELASTICSEARCH_SSL_KEY: /etc/kibana/certs/node/node.key
      #ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES: /etc/kibana/certs/ca/ca.crt
      ELASTICSEARCH_SSL_VERIFICATIONMODE: 'none'

      KIBANA_AUTOCOMPLETETIMEOUT: 3000
      KIBANA_AUTOCOMPLETETERMINATEAFTER: 2500000

      VIS_TYPE_VEGA_ENABLEEXTERNALURLS: 'true'

      XPACK_MAPS_SHOWMAPVISUALIZATIONTYPES: 'true'
      XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY: 'ElastiFlow_0123456789_0123456789_0123456789'

Elasticsearch Node 1

version: '3'
services:
  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
    container_name: elasticsearch
    restart: unless-stopped
    hostname: ES_NODE_NAME_1
    ulimits:
      memlock:
        soft: -1
        hard: -1
      nofile:
        soft: 131072
        hard: 131072
      nproc: 8192
      fsize: -1
    network_mode: bridge
    ports:
      # HTTP/REST
      - 9200:9200/tcp
      # Transport
      - 9300:9300/tcp
    volumes:
      # mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
      - /var/lib/elasticsearch:/usr/share/elasticsearch/data
      - /etc/certs:/usr/share/elasticsearch/config/certificates
    environment:
      ES_JAVA_OPTS: '-Xms31g -Xmx31g'

      cluster.name: elastiflow
      node.name: ES_NODE_NAME_1

      bootstrap.memory_lock: 'true'

      network.bind_host: 0.0.0.0
      network.publish_host: 192.0.2.11

      http.port: 9200
      http.publish_port: 9200

      transport.port: 9300
      transport.publish_port: 9300

      discovery.seed_hosts: '192.0.2.11,192.0.2.12,192.0.2.13'
      cluster.initial_master_nodes: 'ES_NODE_NAME_1,ES_NODE_NAME_2,ES_NODE_NAME_3'

      indices.query.bool.max_clause_count: 8192
      search.max_buckets: 250000

      action.destructive_requires_name: 'true'

      reindex.remote.whitelist: '*:*'
      reindex.ssl.verification_mode: 'none'

      xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
      xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
      xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
      xpack.security.http.ssl.verification_mode: 'none'
      xpack.security.http.ssl.enabled: 'true'

      xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
      xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
      xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
      xpack.security.transport.ssl.verification_mode: 'none'
      xpack.security.transport.ssl.enabled: 'true'

      xpack.monitoring.collection.enabled: 'true'
      xpack.monitoring.collection.interval: 30s

      xpack.security.enabled: 'true'
      xpack.security.audit.enabled: 'false'

Elasticsearch Node 2

version: '3'
services:
  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
    container_name: elasticsearch
    restart: unless-stopped
    hostname: ES_NODE_NAME_2
    ulimits:
      memlock:
        soft: -1
        hard: -1
      nofile:
        soft: 131072
        hard: 131072
      nproc: 8192
      fsize: -1
    network_mode: bridge
    ports:
      # HTTP/REST
      - 9200:9200/tcp
      # Transport
      - 9300:9300/tcp
    volumes:
      # mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
      - /var/lib/elasticsearch:/usr/share/elasticsearch/data
      - /etc/certs:/usr/share/elasticsearch/config/certificates
    environment:
      ES_JAVA_OPTS: '-Xms31g -Xmx31g'

      cluster.name: elastiflow
      node.name: ES_NODE_NAME_2

      bootstrap.memory_lock: 'true'

      network.bind_host: 0.0.0.0
      network.publish_host: 192.0.2.12

      http.port: 9200
      http.publish_port: 9200

      transport.port: 9300
      transport.publish_port: 9300

      discovery.seed_hosts: '192.0.2.11,192.0.2.12,192.0.2.13'
      cluster.initial_master_nodes: 'ES_NODE_NAME_1,ES_NODE_NAME_2,ES_NODE_NAME_3'

      indices.query.bool.max_clause_count: 8192
      search.max_buckets: 250000

      action.destructive_requires_name: 'true'

      reindex.remote.whitelist: '*:*'
      reindex.ssl.verification_mode: 'none'

      xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
      xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
      xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
      xpack.security.http.ssl.verification_mode: 'none'
      xpack.security.http.ssl.enabled: 'true'

      xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
      xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
      xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
      xpack.security.transport.ssl.verification_mode: 'none'
      xpack.security.transport.ssl.enabled: 'true'

      xpack.monitoring.collection.enabled: 'true'
      xpack.monitoring.collection.interval: 30s

      xpack.security.enabled: 'true'
      xpack.security.audit.enabled: 'false'

Elasticsearch Node 3

version: '3'
services:
  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
    container_name: elasticsearch
    restart: unless-stopped
    hostname: ES_NODE_NAME_3
    ulimits:
      memlock:
        soft: -1
        hard: -1
      nofile:
        soft: 131072
        hard: 131072
      nproc: 8192
      fsize: -1
    network_mode: bridge
    ports:
      # HTTP/REST
      - 9200:9200/tcp
      # Transport
      - 9300:9300/tcp
    volumes:
      # mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
      - /var/lib/elasticsearch:/usr/share/elasticsearch/data
      - /etc/certs:/usr/share/elasticsearch/config/certificates
    environment:
      ES_JAVA_OPTS: '-Xms31g -Xmx31g'

      cluster.name: elastiflow
      node.name: ES_NODE_NAME_3

      bootstrap.memory_lock: 'true'

      network.bind_host: 0.0.0.0
      network.publish_host: 192.0.2.13

      http.port: 9200
      http.publish_port: 9200

      transport.port: 9300
      transport.publish_port: 9300

      discovery.seed_hosts: '192.0.2.11,192.0.2.12,192.0.2.13'
      cluster.initial_master_nodes: 'ES_NODE_NAME_1,ES_NODE_NAME_2,ES_NODE_NAME_3'

      indices.query.bool.max_clause_count: 8192
      search.max_buckets: 250000

      action.destructive_requires_name: 'true'

      reindex.remote.whitelist: '*:*'
      reindex.ssl.verification_mode: 'none'

      xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
      xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
      xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
      xpack.security.http.ssl.verification_mode: 'none'
      xpack.security.http.ssl.enabled: 'true'

      xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
      xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
      xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
      xpack.security.transport.ssl.verification_mode: 'none'
      xpack.security.transport.ssl.enabled: 'true'

      xpack.monitoring.collection.enabled: 'true'
      xpack.monitoring.collection.interval: 30s

      xpack.security.enabled: 'true'
      xpack.security.audit.enabled: 'false'