{"type":"doc","content":[{"type":"extension","attrs":{"layout":"default","extensionType":"com.atlassian.confluence.macro.core","extensionKey":"toc","parameters":{"macroParams":{"style":{"value":"none"}},"macroMetadata":{"macroId":{"value":"443d5039-cd0d-4309-98ea-ab97465b7b39"},"schemaVersion":{"value":"1"},"title":"Table of Contents"}},"localId":"f74f0e9a-5803-49ed-a8e8-4db4aa85e3ca"}},{"type":"paragraph"},{"type":"paragraph","content":[{"text":"To enable authentication for ingestion, follow these 3 steps. For step 2 choose the section relevant to the agents being used.","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Step 1. Generate AUTH_TOKEN","type":"text"}]},{"type":"paragraph","content":[{"text":"Generate an auth token (referred to as ","type":"text"},{"text":"AUTH_TOKEN","type":"text","marks":[{"type":"code"}]},{"text":") and store this value in a safe location. You will need to use this later in more than one place.","type":"text"}]},{"type":"codeBlock","content":[{"text":"AUTH_TOKEN=`cat /dev/urandom | env LC_ALL=C tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1`","type":"text"}]},{"type":"paragraph","content":[{"text":"Base64 encode the ","type":"text"},{"text":"AUTH_TOKEN","type":"text","marks":[{"type":"code"}]},{"text":" (referred to as ","type":"text"},{"text":"AUTH_TOKEN_ENCODED","type":"text","marks":[{"type":"code"}]},{"text":" ) ","type":"text"}]},{"type":"codeBlock","content":[{"text":"AUTH_TOKEN_ENCODED=`echo -n $AUTH_TOKEN | base64`","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Step 2. Configure Telemetry agents/sources","type":"text"}]},{"type":"paragraph","content":[{"text":"Follow instructions for the corresponding sources below. ","type":"text"}]},{"type":"orderedList","attrs":{"order":1},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Prometheus Remote Write","type":"text","marks":[{"type":"link","attrs":{"href":"https://kloudfuse.atlassian.net/wiki/spaces/EX/pages/edit-v2/986644493#Prometheus-Remote-Write"}}]},{"text":" ","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Fluent Bit","type":"text","marks":[{"type":"link","attrs":{"href":"https://kloudfuse.atlassian.net/wiki/spaces/EX/pages/986644493"}}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Fluentd","type":"text","marks":[{"type":"link","attrs":{"href":"https://kloudfuse.atlassian.net/wiki/spaces/EX/pages/986644493"}}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Filebeat","type":"text","marks":[{"type":"link","attrs":{"href":"https://kloudfuse.atlassian.net/wiki/spaces/EX/pages/986644493"}}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"OLTP Collector for Metrics/Logs/Traces","type":"text","marks":[{"type":"link","attrs":{"href":"https://kloudfuse.atlassian.net/wiki/spaces/EX/pages/986644493"}}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Datadog/Kfuse agent","type":"text","marks":[{"type":"link","attrs":{"href":"https://kloudfuse.atlassian.net/wiki/spaces/EX/pages/986644493"}}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"AWS CloudWatch metrics","type":"text","marks":[{"type":"link","attrs":{"href":"https://kloudfuse.atlassian.net/wiki/spaces/EX/pages/986644493"}}]},{"text":" & Logs (Kinesis)","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"AWS Eventbridge Events","type":"text","marks":[{"type":"link","attrs":{"href":"https://kloudfuse.atlassian.net/wiki/spaces/EX/pages/986644493"}}]}]}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Prometheus Remote Write","type":"text"}]},{"type":"paragraph","content":[{"text":"Update prometheus remote write configuration as shown below:","type":"text"}]},{"type":"codeBlock","content":[{"text":" prometheus.yml:\n remote_write:\n - url: https://.kloudfuse.io/ingester/write\n authorization:\n credentials: ","type":"text"}]},{"type":"paragraph","content":[{"text":"If you’re using prometheus operator, please refer to the configuration below","type":"text"}]},{"type":"codeBlock","content":[{"text":"remoteWrite:\n - authorization:\n credentials:\n key: authToken\n name: kf-auth-ingest\n url: https://.kloudfuse.io/ingester/write\n","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Fluent Bit","type":"text","marks":[{"type":"annotation","attrs":{"annotationType":"inlineComment","id":"58a9d2c6-084c-409c-a031-221e3db4edd3"}}]}]},{"type":"paragraph","content":[{"text":"Update/Add the following ","type":"text"},{"text":"Headers","type":"text","marks":[{"type":"code"}]},{"text":" field with ","type":"text"},{"text":"AUTH_TOKEN","type":"text","marks":[{"type":"code"}]},{"text":" replaced with the one generated in step 1, in the ","type":"text"},{"text":"HTTP","type":"text","marks":[{"type":"code"}]},{"text":" plugin section of the fluent-bit configuration file as shown below:","type":"text"}]},{"type":"codeBlock","content":[{"text":" [OUTPUT]\n Name http\n Match \n Host \n Port 443\n TLS on\n URI /ingester/v1/fluent_bit\n header Kf-Api-Key ","type":"text"}]},{"type":"heading","attrs":{"level":4},"content":[{"text":"Using Kubernetes Secret for setting the ","type":"text"},{"text":"Kf-Api-Key","type":"text","marks":[{"type":"code"}]}]},{"type":"orderedList","attrs":{"order":1},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Create a secret as following","type":"text"}]}]}]},{"type":"codeBlock","content":[{"text":"apiVersion: v1\nkind: Secret\nmetadata: \n name: <>\ntype: Opaque\ndata:\n AUTH_KEY_ENV: AUTH_TOKEN_ENCODED","type":"text"}]},{"type":"orderedList","attrs":{"order":2},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Update the fluent-bit helm custom-values.yaml as below","type":"text"}]}]}]},{"type":"codeBlock","content":[{"text":"env:\n- name: AUTH_KEY_ENV\n valueFrom:\n secretKeyRef:\n name: <>\n key: AUTH_KEY_ENV\n\n\nconfig:\n outputs: |\n [OUTPUT]\n Name http\n Header Kf-Api-Key ${AUTH_KEY_ENV}\n Match *\n Host swapnil-dev.kloudfuse.io\n Port 443\n TLS on\n URI /ingester/v1/fluent_bit\n Format {json|json_lines|json_stream|msgpack}","type":"text"}]},{"type":"paragraph"},{"type":"heading","attrs":{"level":3},"content":[{"text":"Fluentd","type":"text"}]},{"type":"paragraph","content":[{"text":"Update/Add the fluentd output http plugin configuration to add a “","type":"text"},{"text":"headers","type":"text","marks":[{"type":"code"}]},{"text":"\" field as described below using the ","type":"text"},{"text":"Kf-Api-Key","type":"text","marks":[{"type":"code"}]},{"text":" and ","type":"text"},{"text":"AUTH_TOKEN","type":"text","marks":[{"type":"code"}]},{"text":":","type":"text"}]},{"type":"codeBlock","content":[{"text":" # Match everything\n @type http\n endpoint http://:80/ingester/v1/fluentd\n headers {\"Kf-Api-Key\" : \"\"}\n ...\n","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Filebeat","type":"text"}]},{"type":"paragraph","content":[{"text":"Update/Add the filebeat configuration to include the api_key field within the output section: ","type":"text"}]},{"type":"codeBlock","content":[{"text":"output.elasticsearch: \n hosts: [\"http://:80/ingester/api/v1/filebeat\"]\n api_key: \"\"","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"OLTP Collector for Metrics/Logs/Traces","type":"text"}]},{"type":"paragraph","content":[{"text":"Update/Add following headers section in the exporters section.","type":"text"}]},{"type":"codeBlock","content":[{"text":"exporters:\n otlphttp:\n endpoint: https:///ingester/otlp/metrics\n traces_endpoint: https:///ingester/otlp/traces\n headers:\n kf-api-key: ","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"DD/Kfuse agent","type":"text"}]},{"type":"paragraph","content":[{"text":"Update/Add the dd-agent configuration file to add the ","type":"text"},{"text":"AUTH_TOKEN","type":"text","marks":[{"type":"code"}]},{"text":" as the ","type":"text"},{"text":"apiKey","type":"text","marks":[{"type":"code"}]},{"text":" as shown below:","type":"text"}]},{"type":"codeBlock","content":[{"text":"datadog:\n apiKey: \n ...\n ","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Azure Event Hub","type":"text"}]},{"type":"paragraph","content":[{"text":"Update the ","type":"text"},{"text":"index.js","type":"text","marks":[{"type":"code"}]},{"text":" file in your ","type":"text"},{"text":"FunctionApp's","type":"text","marks":[{"type":"code"}]},{"text":" Trigger Function to add the ","type":"text"},{"text":"AUTH_TOKEN","type":"text","marks":[{"type":"code"}]},{"text":" as the ","type":"text"},{"text":"KF_API_KEY","type":"text","marks":[{"type":"code"}]},{"text":" as shown below:","type":"text"}]},{"type":"codeBlock","content":[{"text":"const KF_API_KEY = process.env.KF_API_KEY || \"\";","type":"text"}]},{"type":"paragraph","content":[{"text":"To configure sending logs to your kloudfuse platform, follow the steps on this wiki ","type":"text"},{"text":"page","type":"text","marks":[{"type":"link","attrs":{"href":"https://kloudfuse.atlassian.net/wiki/spaces/EX/pages/1300365466"}}]},{"text":".","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"AWS CloudWatch metrics & Logs (Kinesis)","type":"text"}]},{"type":"paragraph","content":[{"text":"When configuring ","type":"text"},{"text":"kinesis firehose data stream","type":"text","marks":[{"type":"link","attrs":{"href":"https://kloudfuse.atlassian.net/wiki/spaces/EX/pages/736362497/Collection+of+AWS+CloudWatch+Logs+and+Metrics#Setup-AWS-Kinesis-Firehose"}}]},{"text":" to send logs/metrics from Cloudwatch, use the ","type":"text"},{"text":"AUTH_TOKEN","type":"text","marks":[{"type":"code"}]},{"text":" value generated in step 1 as the ","type":"text"},{"text":"access token","type":"text","marks":[{"type":"code"}]},{"text":". If the firehose data stream is already setup, then update it to use ","type":"text"},{"text":"AUTH_TOKEN","type":"text","marks":[{"type":"code"}]},{"text":" value as ","type":"text"},{"text":"access token","type":"text","marks":[{"type":"code"}]},{"text":".","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"AWS Eventbridge Events","type":"text"}]},{"type":"paragraph","content":[{"text":"When ","type":"text"},{"text":"configuring Eventbridge","type":"text","marks":[{"type":"link","attrs":{"href":"https://kloudfuse.atlassian.net/wiki/spaces/EX/pages/748683276/Collection+of+AWS+Events#Create-EventBridge-Connection"}}]},{"text":" to ingest to Kloudfuse, use ","type":"text"},{"text":"AUTH_TOKEN","type":"text","marks":[{"type":"code"}]},{"text":" as the value for the ","type":"text"},{"text":"Kf_Api_Key","type":"text","marks":[{"type":"code"}]},{"text":". ","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Step 3: Configure kfuse","type":"text"}]},{"type":"paragraph","content":[{"text":"Use the base64 encoded value of the ","type":"text"},{"text":"AUTH_TOKEN","type":"text","marks":[{"type":"code"}]},{"text":" (","type":"text"},{"text":"AUTH_TOKEN_ENCODED","type":"text","marks":[{"type":"code"}]},{"text":") and create a kubernetes secret with the name ","type":"text"},{"text":"kfuse-auth-ingest","type":"text","marks":[{"type":"code"}]},{"text":":","type":"text"}]},{"type":"codeBlock","content":[{"text":"apiVersion: v1\nkind: Secret\nmetadata: \n name: kfuse-auth-ingest\ntype: Opaque\ndata:\n authToken: ","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Multiple Authorization Keys","type":"text"}]},{"type":"panel","attrs":{"panelIconId":"a63e4601-9a66-470d-901e-7e5d511e5403","panelIcon":":new:","panelIconText":":new:","panelColor":"#DEEBFF","panelType":"custom"},"content":[{"type":"paragraph","content":[{"text":"New feature available in Release 2.7.2","type":"text"}]}]},{"type":"paragraph","content":[{"text":"You can configure multiple authorization tokens inside the secret. These tokens can contain any string value, and can be used as a human-readable identifier to reference the auth token.","type":"text"}]},{"type":"paragraph","content":[{"text":"Here, line 8 specifies the second key, ","type":"text"},{"text":"authkey2","type":"text","marks":[{"type":"code"}]},{"text":".","type":"text"}]},{"type":"codeBlock","content":[{"text":"apiVersion: v1\nkind: Secret\nmetadata: \n name: kfuse-auth-ingest\ntype: Opaque\ndata:\n authkey1: \n authkey2: ","type":"text"}]},{"type":"paragraph","content":[{"text":"Remember to update the ","type":"text"},{"text":"custom-values.yaml","type":"text","marks":[{"type":"code"}]},{"text":" file to enable ingestion authentication:","type":"text"}]},{"type":"codeBlock","content":[{"text":"global:\n authConfig:\n enabled: true","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"type":"hardBreak"},{"text":"Configuration of Additional Labels based on Auth Token","type":"text"}]},{"type":"paragraph","content":[{"text":"From Kloudfuse 2.7.2 onwards, additional labels can be configured to be attached to MELT data based on the auth token used by the incoming payload.","type":"text"}]},{"type":"paragraph","content":[{"text":"To configure these labels, add the following in the custom-values.yaml (replace accordingly). Take note that the same keys are used in the secret and the ingester config.","type":"text"}]},{"type":"codeBlock","content":[{"text":"ingester:\n config:\n authKeyAdditionalLabels:\n authkey1:\n - name: label1\n value: val1\n - name: label2\n value: val2\n authkey2:\n - name: label1\n value: val3\n - name: label4\n value: val4 ","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Restart the ingester post the auth-token changes","type":"text"}]},{"type":"codeBlock","content":[{"text":"kubectl -n kfuse rollout restart sts ingester","type":"text"}]}],"version":1}

Browser not supported