{"type":"doc","content":[{"type":"heading","attrs":{"level":2},"content":[{"text":"Add Extension Layers to the Lambda Function","type":"text"}]},{"type":"paragraph","content":[{"text":"Kloudfuse requires two extension layers to be added to the lambda function.","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Datadog Extension Layer","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Kloudfuse is tested with version 33","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Layer ARN: ","type":"text"},{"text":"arn:aws:lambda:us-west-2:464622532012:layer:Datadog-Extension:33","type":"text","marks":[{"type":"code"}]}]}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"LambdaInsightsExtension","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Kloudfuse is tested with version 21","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Layer ARN: ","type":"text"},{"text":"arn:aws:lambda:us-west-2:580247275435:layer:LambdaInsightsExtension:21","type":"text","marks":[{"type":"code"}]}]}]}]}]}]},{"type":"paragraph","content":[{"text":"The layers can be added in the AWS Lambda console of the Lambda function","type":"text"}]},{"type":"mediaSingle","attrs":{"layout":"center","width":760,"widthType":"pixel"},"content":[{"type":"media","attrs":{"width":1438,"alt":"Screenshot 2024-08-16 at 3.39.57 PM.png","id":"54248ca1-b785-4f4a-9b34-24fb79875877","collection":"contentId-1152843789","type":"file","height":207}}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Configure Datadog Extension Layer","type":"text"}]},{"type":"paragraph","content":[{"text":"Add the following environment variables in the Lambda configuration","type":"text"}]},{"type":"table","attrs":{"layout":"default","width":760.0,"localId":"d2756dca-b47e-4f1c-a860-23279a7aab61"},"content":[{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1},"content":[{"type":"paragraph","content":[{"text":"DD_API_KEY","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1},"content":[{"type":"paragraph","content":[{"text":"If ","type":"text"},{"text":"authenticated ingest","type":"text","marks":[{"type":"link","attrs":{"href":"https://kloudfuse.atlassian.net/wiki/spaces/EX/pages/986644493"}}]},{"text":" is enabled, provide the configured auth token. Otherwise, provide any string. ","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1},"content":[{"type":"paragraph","content":[{"text":"DD_APM_DD_URL","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1},"content":[{"type":"paragraph","content":[{"text":"https:///ingester","type":"text","marks":[{"type":"code"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1},"content":[{"type":"paragraph","content":[{"text":"DD_DD_URL","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1},"content":[{"type":"paragraph","content":[{"text":"https:///ingester","type":"text","marks":[{"type":"code"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1},"content":[{"type":"paragraph","content":[{"text":"DD_LOGS_CONFIG_LOGS_DD_URL","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1},"content":[{"type":"paragraph","content":[{"text":":443","type":"text","marks":[{"type":"code"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1},"content":[{"type":"paragraph","content":[{"text":"DD_LOGS_CONFIG_LOGS_NO_SSL","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1},"content":[{"type":"paragraph","content":[{"text":"false","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1},"content":[{"type":"paragraph","content":[{"text":"DD_LOGS_CONFIG_USE_V2_API","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1},"content":[{"type":"paragraph","content":[{"text":"false","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1},"content":[{"type":"paragraph","content":[{"text":"DD_TRACE_ENABLED","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1},"content":[{"type":"paragraph","content":[{"text":"true","type":"text"}]}]}]}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Configure Cloudwatch Metrics","type":"text"}]},{"type":"paragraph","content":[{"text":"Lambda-related metrics are pushed to Cloudwatch by default. Refer to ","type":"text"},{"text":"Cloudwatch integration","type":"text","marks":[{"type":"link","attrs":{"href":"https://kloudfuse.atlassian.net/wiki/spaces/EX/pages/736362497"}}]},{"text":" to push Cloudwatch metrics to Kloudfuse.","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Configure EventBridge and CloudTrail","type":"text"}]},{"type":"paragraph","content":[{"text":"Lambda-related events can be integrated with Kloudfuse using EventBridge. Refer to ","type":"text"},{"text":"Eventbridge integration","type":"text","marks":[{"type":"link","attrs":{"href":"https://kloudfuse.atlassian.net/wiki/spaces/EX/pages/748683276"}}]},{"text":" to push AWS events to Kloudfuse.","type":"text"}]},{"type":"paragraph","content":[{"text":"CloudTrail needs to be configured to send Lambda events to the EventBridge.","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Create a new Trail from the AWS CloudTrail console","type":"text"}]},{"type":"mediaSingle","attrs":{"layout":"center","width":736,"widthType":"pixel"},"content":[{"type":"media","attrs":{"width":1432,"alt":"Screenshot 2024-08-16 at 3.54.44 PM.png","id":"8480cb6d-e096-4e24-ad5d-a214a23c043c","collection":"contentId-1152843789","type":"file","height":140}}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"In the ","type":"text"},{"text":"Choose log events","type":"text","marks":[{"type":"code"}]},{"text":" step, select ","type":"text"},{"text":"Data events","type":"text","marks":[{"type":"code"}]},{"text":" and ","type":"text"},{"text":"Lambda","type":"text","marks":[{"type":"code"}]},{"text":" as ","type":"text"},{"text":"Data event type","type":"text","marks":[{"type":"code"}]},{"text":".","type":"text"}]},{"type":"mediaSingle","attrs":{"layout":"center","width":736,"widthType":"pixel"},"content":[{"type":"media","attrs":{"width":1105,"alt":"Screenshot 2024-08-16 at 3.57.08 PM.png","id":"e66a8db7-7cc9-4b53-a763-92f4325fa4ef","collection":"contentId-1152843789","type":"file","height":753}}]},{"type":"paragraph","content":[{"text":" ","type":"text"}]}]}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Update Helm Values","type":"text"}]},{"type":"orderedList","attrs":{"order":1},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"In ","type":"text"},{"text":"custom_values.yaml","type":"text","marks":[{"type":"code"}]},{"text":" enable lambda enrichment.","type":"text"}]}]}]},{"type":"codeBlock","content":[{"text":"ingester:\n config:\n awsScrapeLambdaConfigs: true","type":"text"}]},{"type":"orderedList","attrs":{"order":2},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Since Kfuse needs to scrape Lambda configuration from AWS, Kfuse requires a policy with the following permissions:","type":"text"}]},{"type":"codeBlock","content":[{"text":"\"Action\": [\n\t\t\t\t\"lambda:GetPolicy\",\n\t\t\t\t\"lambda:List*\",\n\t\t\t\t\"lambda:ListTags\",\n\t\t\t]","type":"text"}]}]}]},{"type":"panel","attrs":{"panelType":"note"},"content":[{"type":"paragraph","content":[{"text":"Please make sure the permissions mapped to the correct nodepool being used for EKS cluster where Kloudfuse is hosted.","type":"text"}]}]},{"type":"heading","attrs":{"level":4},"marks":[{"type":"indentation","attrs":{"level":1}}],"content":[{"text":"Step 3.1","type":"text","marks":[{"type":"strong"}]},{"text":": Create an IAM scraper role with a policy to allow scraping on AWS labels.","type":"text"}]},{"type":"paragraph","marks":[{"type":"indentation","attrs":{"level":1}}],"content":[{"text":"Follow the instructions on the AWS page to create an IAM policy: ","type":"text"},{"type":"inlineCard","attrs":{"url":"https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_create.html"}},{"text":" ","type":"text"}]},{"type":"heading","attrs":{"level":4},"marks":[{"type":"indentation","attrs":{"level":1}}],"content":[{"text":"Step 3.2: Use one of the following options for kfuse to consume the policy created above","type":"text","marks":[{"type":"strong"}]}]},{"type":"heading","attrs":{"level":5},"marks":[{"type":"indentation","attrs":{"level":1}}],"content":[{"text":"Option 1","type":"text","marks":[{"type":"strong"}]},{"text":": Add your AWS credentials as a secret and use the secret in the ingester config.","type":"text"}]},{"type":"paragraph","marks":[{"type":"indentation","attrs":{"level":1}}],"content":[{"text":"You can retrieve your aws credentials required for the next step ","type":"text"},{"text":"here","type":"text","marks":[{"type":"link","attrs":{"href":"https://docs.aws.amazon.com/powershell/latest/userguide/pstools-appendix-sign-up.html"}}]},{"text":".","type":"text"}]},{"type":"paragraph","marks":[{"type":"indentation","attrs":{"level":1}}],"content":[{"text":"Create a kube secret name named “aws-access-key” with keys “accessKey” and “secretKey” in the kfuse namespace","type":"text"}]},{"type":"codeBlock","content":[{"text":"kubectl create secret generic aws-access-key --from-literal=accessKey= --from-literal=secretKey=","type":"text"}]},{"type":"paragraph","marks":[{"type":"indentation","attrs":{"level":1}}],"content":[{"text":"Specify the secretName in the custom values.yaml.","type":"text"}]},{"type":"codeBlock","content":[{"text":"ingester:\n config:\n awsScraper:\n secretName: aws-access-key","type":"text"}]},{"type":"paragraph","marks":[{"type":"indentation","attrs":{"level":1}}],"content":[{"text":"Kfuse by default attempts to scrape from all regions. This can be customized by adding the following configuration in the custom values.yaml","type":"text"}]},{"type":"codeBlock","content":[{"text":"ingester:\n config:\n awsScraper:\n secretName: aws-access-key\n regions:\n - ","type":"text"}]},{"type":"paragraph","marks":[{"type":"indentation","attrs":{"level":1}}],"content":[{"text":"Do a helm upgrade for changes to take affect","type":"text"}]},{"type":"codeBlock","content":[{"text":"helm upgrade --create-namespace --install kfuse . -f ","type":"text"}]},{"type":"heading","attrs":{"level":5},"marks":[{"type":"indentation","attrs":{"level":1}}],"content":[{"text":"Option 2: Add Role ARNs in the ingester config.","type":"text"}]},{"type":"paragraph","marks":[{"type":"indentation","attrs":{"level":1}}],"content":[{"text":"With this option, Kfuse can be configured to scrape multiple AWS accounts.","type":"text"}]},{"type":"paragraph","marks":[{"type":"indentation","attrs":{"level":1}}],"content":[{"text":"Add the scraper Role ARNs (created with the permissions above) in the awsRoleArns list to your custom values.yaml","type":"text"}]},{"type":"codeBlock","content":[{"text":"ingester:\n config:\n awsRoleArns:\n - role: ","type":"text"}]},{"type":"paragraph","marks":[{"type":"indentation","attrs":{"level":1}}],"content":[{"text":"Kfuse by default attempts to scrape from all regions. This can be customized by adding the following configuration in the custom values.yaml","type":"text"}]},{"type":"codeBlock","content":[{"text":"ingester:\n config:\n awsRoleArns:\n - role: \n regions:\n - ","type":"text"}]},{"type":"paragraph","marks":[{"type":"indentation","attrs":{"level":1}}],"content":[{"text":"If needed modify the Trust Relationship for the policy of the scrape role ARN to add the node-group (Node IAM Role ARN), in which Kloudfuse is running on, as the Principal on the Account.","type":"text"}]},{"type":"codeBlock","content":[{"text":"{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"Statement1\",\n \"Effect\": \"Allow\",\n \"Principal\": {\n \"AWS\": \"arn:aws:iam::ACCOUNT-NUMBER:role/eksctl-XXXXX-nodegroup-ng-XXXXXX-NodeInstanceRole-XXXXXXXXXX\"\n },\n \"Action\": \"sts:AssumeRole\"\n }\n ]\n}","type":"text"}]},{"type":"orderedList","attrs":{"order":4},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"The node-group (Node IAM Role ARN), in which Kloudfuse is running on, also needs to have the following permissions policy to assume the role.","type":"text"}]}]}]},{"type":"codeBlock","content":[{"text":"{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Resource\": \n } \n ]\n}","type":"text"}]},{"type":"paragraph"},{"type":"paragraph","content":[{"text":"Do a helm upgrade for changes to take affect","type":"text"}]},{"type":"codeBlock","content":[{"text":"helm upgrade --create-namespace --install kfuse . -f ","type":"text"}]}],"version":1}

Browser not supported