rel-nifi.yaml

apiVersion: nifi.orange.com/v1alpha1
kind: NifiCluster
metadata:
  name: mynifi
spec:
  service:
    # headlessEnabled specifies if the cluster should use headlessService for Nifi or individual services
    # using service per nodes may come an handy case of service mesh.
    headlessEnabled: true
    # Annotations specifies the annotations to attach to services the operator creates
    #    annotations:
    #  pod:
    # Annotations specifies the annotations to attach to pods the operator creates
  #    annotations:
  # zKAddress specifies the ZooKeeper connection string
  # in the form hostname:port where host and port are those of a Zookeeper server.
  zkAddress: "zookeeper.zookeeper:2181"
  # zKPath specifies the Zookeeper chroot path as part
  # of its Zookeeper connection string which puts its data under same path in the global ZooKeeper namespace.
  zkPath: "/myNiFi"
  # initContainerImage can override the default image used into the init container to check if
  # ZoooKeeper server is reachable.
  #  initContainerImage: busybox
  # initContainers defines additional initContainers configurations
  initContainers:
   - image: eu.gcr.io/poc-rtc/nifi-proxydata-bundle:1.11.4-1.0.0
     name: nifi-proxydata-lib
     imagePullPolicy: IfNotPresent
     command: ["sh", "-c", "cp -vr /nifi_lib/* /nifi_lib_volume/"]
     volumeMounts:
       - name: extensions-repository
         mountPath: /nifi_lib_volume
       - name: logs
         mountPath: /logs
  # clusterImage can specify the whole nificluster image in one place
  clusterImage: "apache/nifi:1.12.1"
  # oneNifiNodePerNode if set to true every nifi node is started on a new node, if there is not enough node to do that
  # it will stay in pending state. If set to false the operator also tries to schedule the nifi node to a unique node
  # but if the node number is insufficient the nifi node will be scheduled to a node where a nifi node is already running.
  oneNifiNodePerNode: false
  #
  propagateLabels: true
  #  managedAdminUsers:
  #    -  identity : "alexandre.guitton@orange.com"
  #       name: "aguitton"
  #  managedReaderUsers:
  #    -  identity : "toto@orange.com"
  #       name: "toto"
  # readOnlyConfig specifies the read-only type Nifi config cluster wide, all theses
  # will be merged with node specified readOnly configurations, so it can be overwritten per node.
  readOnlyConfig:
    # MaximumTimerDrivenThreadCount define the maximum number of threads for timer driven processors available to the system.
    maximumTimerDrivenThreadCount: 30
    # Logback configuration that will be applied to the node
    logbackConfig:
      # logback.xml configuration that will replace the one produced based on template
      replaceConfigMap:
        # The key of the value,in data content, that we want use.
        data: logback.xml
        # Name of the configmap that we want to refer.
        name: nifi-config
        # Namespace where is located the secret that we want to refer.
        namespace: nifi

    # NifiProperties configuration that will be applied to the node.
    nifiProperties:
      # Additionnals nifi.properties configuration that will override the one produced based on template and
      # configuration
      overrideConfigMap:
        # The key of the value,in data content, that we want use.
        data: nifi.properties
        # Name of the configmap that we want to refer.
        name: nifi-config
        # Namespace where is located the secret that we want to refer.
        namespace: nifi
      # A comma separated list of allowed HTTP Host header values to consider when NiFi
      # is running securely and will be receiving requests to a different host[:port] than it is bound to.
      # https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#web-properties
      #      webProxyHosts:
      # Nifi security client auth
      needClientAuth: false
      # Indicates which of the configured authorizers in the authorizers.xml file to use
      # https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#authorizer-configuration
    #      authorizer:
    # ZookeeperProperties configuration that will be applied to the node.
    zookeeperProperties:
     # Additionnals zookeeeper.properties configuration that will override the one produced based on template and
     # configuration
     overrideConfigMap:
       # The key of the value,in data content, that we want use.
       data: zookeeper.properties
       # Name of the configmap that we want to refer.
       name: nifi-config
       # Namespace where is located the secret that we want to refer.
       namespace: nifi

    bootstrapProperties:
     # Additionnals bootstrap.properties configuration that will override the one produced based on template and
     # configuration
     overrideConfigMap:
       # The key of the value,in data content, that we want use.
       data: bootstrap.conf
       # Name of the configmap that we want to refer.
       name: nifi-config
       # Namespace where is located the secret that we want to refer.
       namespace: nifi

      # JVM memory settings
    nifiJvmMemory: "512m"
      # Additionnals bootstrap.properties configuration that will override the one produced based
      # on template and configurations.
      # https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#bootstrap_properties
    overrideConfigs: |
      java.arg.4=-Djava.net.preferIPv4Stack=true
  # nodeConfigGroups specifies multiple node configs with unique name
  nodeConfigGroups:
    default_group:
      # provenanceStorage allow to specify the maximum amount of data provenance information to store at a time
      # https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#write-ahead-provenance-repository-properties
      provenanceStorage: "10 GB"
      #RunAsUser define the id of the user to run in the Nifi image
      # +kubebuilder:validation:Minimum=1
      runAsUser: 1000
      # Set this to true if the instance is a node in a cluster.
      # https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#basic-cluster-setup
      isNode: true
      # Docker image used by the operator to create the node associated
      # https://hub.docker.com/r/apache/nifi/
      #      image: "apache/nifi:1.11.2"
      # nodeAffinity can be specified, operator populates this value if new pvc added later to node
      # https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#node-affinity
      #      nodeAffinity:
      # imagePullPolicy define the pull policy for NiFi cluster docker image
      imagePullPolicy: IfNotPresent
      # storageConfigs specifies the node related configs
      storageConfigs:
        # Name of the storage config, used to name PV to reuse into sidecars for example.
        - name: provenance-repository
          # Path where the volume will be mount into the main nifi container inside the pod.
          mountPath: "/opt/nifi/provenance_repository"
          # Kubernetes PVC spec
          # https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim
          pvcSpec:
            accessModes:
              - ReadWriteMany
            storageClassName: "ibmc-file-silver"
            resources:
              requests:
                storage: 10Gi
        - mountPath: "/opt/nifi/nifi-current/logs"
          name: logs
          pvcSpec:
            accessModes:
              - ReadWriteMany
            storageClassName: "ibmc-file-silver"
            resources:
              requests:
                storage: 10Gi
      # serviceAccountName specifies the serviceAccount used for this specific node
      serviceAccountName: "default"
      # resourceRequirements works exactly like Container resources, the user can specify the limit and the requests
      # through this property
      # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/
      resourcesRequirements:
        limits:
          cpu: "2"
          memory: 3Gi
        requests:
          cpu: "1"
          memory: 1Gi
      # imagePullSecrets specifies the secret to use when using private registry
      # https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#localobjectreference-v1-core
      # imagePullSecrets: devops-registry-secret
      # nodeSelector can be specified, which set the pod to fit on a node
      # https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
      #      nodeSelector:
      # tolerations can be specified, which set the pod's tolerations
      # https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/#concepts
      #      tolerations:
      # Additionnal annotation to attach to the pod associated
      # https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set
  #      nodeAnnotations:
  # disruptionBudget defines the configuration for PodDisruptionBudget
  disruptionBudget:
    # create will enable the PodDisruptionBudget when set to true
    create: false
  # The budget to set for the PDB, can either be static number or a percentage
  #   budget: "1"
  # all node requires an image, unique id, and storageConfigs settings

  nodes:
    # Unique Node id
    - id: 0
      # nodeConfigGroup can be used to ease the node configuration, if set only the id is required
      nodeConfigGroup: "default_group"
      # readOnlyConfig can be used to pass Nifi node config
      # which has type read-only these config changes will trigger rolling upgrade
      readOnlyConfig:
        nifiProperties:
          overrideConfigs: |
            nifi.ui.banner.text=NiFiKop by Orange - Node 0
      # node configuration
    #       nodeConfig:
    - id: 1
      # nodeConfigGroup can be used to ease the node configuration, if set only the id is required
      nodeConfigGroup: "default_group"
      # readOnlyConfig can be used to pass Nifi node config
      # which has type read-only these config changes will trigger rolling upgrade
      readOnlyConfig:
        nifiProperties:
          overrideConfigs: |
            nifi.ui.banner.text=NiFiKop by Orange - Node 1
      # node configuration
    #       nodeConfig:
    - id: 2
      # readOnlyConfig can be used to pass Nifi node config
      # which has type read-only these config changes will trigger rolling upgrade
      nodeConfigGroup: "default_group"
      readOnlyConfig:
        overrideConfigs: |
          nifi.ui.banner.text=NiFiKop by Orange - Node 2

  # LdapConfiguration specifies the configuration if you want to use LDAP
  ldapConfiguration:
    # If set to true, we will enable ldap usage into nifi.properties configuration.
       enabled: true
    # Space-separated list of URLs of the LDAP servers (i.e. ldap://<hostname>:<port>).
       url: ldaps://bluepages.ibm.com:636
    # Base DN for searching for users (i.e. CN=Users,DC=example,DC=com).
       searchBase: ou=bluepages,o=ibm.com
    # Filter for searching for users against the 'User Search Base'.
    # (i.e. sAMAccountName={0}). The user specified name is inserted into '{0}'.
       searchFilter: (mail={0})
       groupSearchBase: ou=memberlist,ou=ibmgroups,o=ibm.com
       groupObjectClass: groupOfUniqueNames
       groupSearchFilter: (cn=Optimizer_SVT*)
       groupMemberAttribute: uniquemember
       groupNameAttribute: cn
       admin: uid=AVTMH6672,c=cn,ou=bluepages,o=ibm.com
       initUser: cn=Optimizer_SVT_Write,ou=memberlist,ou=ibmgroups,o=ibm.com

  # NifiClusterTaskSpec specifies the configuration of the nifi cluster Tasks
  nifiClusterTaskSpec:
    # retryDurationMinutes describes the amount of time the Operator waits for the task
    retryDurationMinutes: 10

  # listenerConfig specifies nifi's listener specifig configs
  listenersConfig:
    # internalListeners specifies settings required to access nifi internally
    internalListeners:
      # (Optional field) Type allow to specify if we are in a specific nifi listener
      # it's allowing to define some required information such as Cluster Port,
      # Http Port, Https Port or S2S port
      # {"cluster", "http", "https", "s2s"}
      - type: "https"
        # An identifier for the port which will be configured.
        name: "https"
        # The container port.
        containerPort: 8080
      - type: "cluster"
        name: "cluster"
        containerPort: 6007
      - type: "s2s"
        name: "s2s"
        containerPort: 10000
      - name: "prometheus"
        containerPort: 9020
      - name: "rsyslog"
        containerPort: 10001
      - type: "prometheus"
        name: "prometheus"
        containerPort: 9090
      # sslSecrets contains information about ssl related kubernetes secrets if one of the
      # listener setting type set to ssl these fields must be populated to
    sslSecrets:
      tlsSecretName: "nifi-ssl-tls-secret"
      #      tlsSecretName: "test-nifikop"
      # create tells the installed cert manager to create the required certs keys
      create: true
      clusterScoped: false
      #      clusterScoped: false
      # issuerRef allow to use an existing issuer to act as CA :
      #	https://cert-manager.io/docs/concepts/issuer/ to-do
#      issuerRef:
#        name: letsencrypt-staging
#        kind: Issuer
  externalServices:
    - name: "clusterip"
      spec:
        type: ClusterIP
        portConfigs:
          - port: 8080
            internalListenerName: "https"
      serviceAnnotations:
        toto: tata
  sidecarConfigs:
    - name: app-log
      image: "busybox:1.32.0"
      args: [ tail, -n+1, -F, /var/log/nifi-app.log ]
      resources: &log_resources
        requests:
          cpu: 50m
          memory: 50Mi
        limits:
          cpu: 50m
          memory: 50Mi
        volumeMounts:
          - name: logs
            mountPath: /var/log
    - name: bootstrap-log
      image: "busybox:1.32.0"
      args: [tail, -n+1, -F, /var/log/nifi-bootstrap.log]
      resources: *log_resources
      volumeMounts:
        - name: logs
          mountPath: /var/log
    - name: user-log
      image: "busybox:1.32.0"
      args: [tail, -n+1, -F, /var/log/nifi-user.log]
      resources: *log_resources
      volumeMounts:
        - name: logs
          mountPath: /var/log