{"type":"doc","content":[{"type":"extension","attrs":{"layout":"default","extensionType":"com.atlassian.confluence.macro.core","extensionKey":"toc","parameters":{"macroParams":{"minLevel":{"value":"1"},"maxLevel":{"value":"6"},"outline":{"value":"false"},"style":{"value":"default"},"type":{"value":"list"},"printable":{"value":"true"}},"macroMetadata":{"macroId":{"value":"8193b2530130c978a6dd6553fec4d2a5"},"schemaVersion":{"value":"1"},"title":"Table of Contents"}},"localId":"1a800517-7ce7-477c-927f-a33e81e7d8be"}},{"type":"heading","attrs":{"level":1},"content":[{"text":"Assumptions","type":"text"}]},{"type":"paragraph"},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"If using an existing dd-agent version (and just pointing it to KloudFuse instead of DD HQ)","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"dd agent version needs to be 7.41 or higher.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"validate using following steps:","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"check the chart version with helm list - should be 3.1.10 or higher ","type":"text"}]},{"type":"codeBlock","content":[{"text":"helm list -n ","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"check the image version of the agent with describe pod on dd-agent pod. ","type":"text"}]},{"type":"codeBlock","content":[{"text":"k describe pod -n | grep Image\n Image: gcr.io/datadoghq/agent:7.36.0","type":"text"}]}]}]}]}]}]}]},{"type":"heading","attrs":{"level":1},"content":[{"text":"Install DataDog Agent","type":"text"}]},{"type":"table","attrs":{"layout":"full-width","width":1800.0,"localId":"f0ac1a3f-b39b-457f-b700-16d01b8a25ac"},"content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":[607.0]},"content":[{"type":"paragraph","content":[{"text":"Agent install Setup Scenario","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":[1109.0]},"content":[{"type":"paragraph","content":[{"text":"Steps","type":"text","marks":[{"type":"strong"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[607.0]},"content":[{"type":"paragraph","content":[{"text":"Kloudfuse stack & target both in same VPC and in same K8S cluster (Default)","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[1109.0]},"content":[{"type":"paragraph","content":[{"text":"This is the default scenario. Just use the provided ","type":"text"},{"text":"values file","type":"text","marks":[{"type":"link","attrs":{"href":"https://kloudfuse.atlassian.net/wiki/pages/viewpageattachments.action?pageId=753991796#:~:text=dd%2Dvalues%2Dkfuse%2Dcustomer.yaml"}}]},{"text":" for the agent and ","type":"text"},{"text":"install","type":"text","marks":[{"type":"link","attrs":{"href":"https://kloudfuse.atlassian.net/wiki/spaces/EX/pages/753991796/Datadog+Agent+setup+for+Kubernetes#Install-command"}}]},{"text":".","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[607.0]},"content":[{"type":"paragraph","content":[{"text":"Kloudfuse stack & target both in same VPC, but in different K8S cluster","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[1109.0]},"content":[{"type":"paragraph","content":[{"text":"Search for ","type":"text"},{"text":"_url","type":"text","marks":[{"type":"code"}]},{"text":" in the provided file. Wherever found, comment the ","type":"text"},{"text":"Default Scenario","type":"text","marks":[{"type":"code"}]},{"text":" and uncomment ","type":"text"},{"text":"Scenario 1","type":"text","marks":[{"type":"code"}]},{"text":". For the IP address values needed, please see the following steps.","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Get ingress internal IP ","type":"text"}]},{"type":"codeBlock","content":[{"text":"kubectl get svc -n kfuse | grep kfuse-ingress-nginx-controller-internal\nkfuse-ingress-nginx-controller-internal LoadBalancer 10.53.250.80 10.53.232.3 80:32716/TCP,443:30767/TCP 125m","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"replace all settings with ","type":"text"},{"text":"_url","type":"text","marks":[{"type":"code"}]},{"text":" suffix, for example:","type":"text"}]},{"type":"codeBlock","content":[{"text":"dd_url: http://10.53.250.80/ingester\nlogs_dd_url: \"10.53.250.80:80\"","type":"text"}]}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[607.0]},"content":[{"type":"paragraph","content":[{"text":"Kloudfuse stack hosted in a different VPC (hosted at “","type":"text","marks":[{"type":"strong"}]},{"text":"customer.kloudfuse.io","type":"text","marks":[{"type":"strong"},{"type":"link","attrs":{"href":"http://customer.kloudfuse.io"}}]},{"text":"”)","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[1109.0]},"content":[{"type":"paragraph","content":[{"text":"Search for ","type":"text"},{"text":"_url","type":"text","marks":[{"type":"code"}]},{"text":" in the values.yaml file provided in the first scenario. Wherever found, comment the ","type":"text"},{"text":"Default Scenario","type":"text","marks":[{"type":"code"}]},{"text":" and uncomment ","type":"text"},{"text":"Scenario 2","type":"text","marks":[{"type":"code"}]},{"text":". Replace the ","type":"text"},{"text":"customer.kloudfuse.io","type":"text","marks":[{"type":"code"}]},{"text":" with your custom DNS name entry.","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"replace all settings with ","type":"text"},{"text":"_url","type":"text","marks":[{"type":"code"}]},{"text":" suffix, for example:","type":"text"}]},{"type":"codeBlock","content":[{"text":"dd_url: http://customer.kloudfuse.io/ingester\nlogs_dd_url: \"customer.kloudfuse.io:443\"","type":"text"}]}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[607.0]},"content":[{"type":"paragraph"}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[1109.0]},"content":[{"type":"paragraph"}]}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Collect high cardinality tags","type":"text"}]},{"type":"paragraph","content":[{"text":"The Datadog Agent is installed with cardinality set to ‘orchestrator’ level - allowing granular (pod and container level) ","type":"text"},{"text":"tagging of metrics","type":"text","marks":[{"type":"link","attrs":{"href":"https://docs.datadoghq.com/getting_started/tagging/assigning_tags/?tab=noncontainerizedenvironments#tags-cardinality"}}]},{"text":".. The default setting in the Datadog Agent is set to ‘low’ - allowing tagging only at host level.","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Install command","type":"text"}]},{"type":"paragraph","content":[{"text":"If you haven’t before, add datadog helm repo:","type":"text"}]},{"type":"codeBlock","content":[{"text":"helm repo add datadog https://helm.datadoghq.com\nhelm repo update","type":"text"}]},{"type":"paragraph","content":[{"text":"Install the datadog agent with updated values (cmd below assumes the file is called dd-values-kfuse.yaml. Replace that argument with the filename that’s relevant to your scenario).","type":"text"}]},{"type":"codeBlock","content":[{"text":"# Create separate namespace for the agent to be installed (if required)\nkubectl create namespace kfuse-agent\nhelm upgrade --install kfuse-agent -f dd-values-kfuse.yaml datadog/datadog -n kfuse-agent --version 3.6.7","type":"text"}]},{"type":"heading","attrs":{"level":1},"content":[{"text":"Adding custom tags","type":"text"}]},{"type":"paragraph","content":[{"text":"Custom tags can be added to the agent such that all metrics collected by the agent will have the tag added. To add custom tags, please follow these steps. ","type":"text"}]},{"type":"orderedList","attrs":{"order":1},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Update ","type":"text"},{"text":"dd-values-kfuse.yaml","type":"text","marks":[{"type":"code"}]},{"text":" file to include following for the custom tags (","type":"text"},{"text":"please note that each tag should be added as ","type":"text","marks":[{"type":"strong"}]},{"text":"key:value","type":"text","marks":[{"type":"code"}]},{"text":" as a string separated by the \"","type":"text","marks":[{"type":"strong"}]},{"text":":","type":"text","marks":[{"type":"code"}]},{"text":"\"","type":"text","marks":[{"type":"strong"}]},{"text":") :","type":"text"}]}]}]},{"type":"codeBlock","content":[{"text":"datadog:\n tags:\n - custom_tag_name:custom_tag_value","type":"text"}]},{"type":"orderedList","attrs":{"order":2},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"kfuse helm","type":"text"},{"text":"custom-values.yaml","type":"text","marks":[{"type":"code"}]},{"text":" file needs to be updated to whitelist the new custom tag (note that the following entry needs to be “appended” to the existing list, without which some default values will get overwritten which is not the desired behavior. Please ","type":"text"},{"text":"contact the Kloudfuse","type":"text","marks":[{"type":"link","attrs":{"href":"https://kloudfuse.atlassian.net/wiki/spaces/EX/pages/760774669"}}]},{"text":" team for assistance):","type":"text"}]}]}]},{"type":"codeBlock","content":[{"text":"ingester:\n config:\n hostTagIncludes:\n - kf\n - kfuse\n - kube_cluster_name\n - kubernetes.io/hostname\n - node.kubernetes.io/instance-type\n - org_id\n - project\n - topology.kubernetes.io/region\n - topology.kubernetes.io/zone\n ...\n - custom_tag_name","type":"text"}]},{"type":"heading","attrs":{"level":1},"content":[{"text":"Enabling Pods to be detected by Prometheus Autodiscovery","type":"text"}]},{"type":"paragraph","content":[{"text":"In addition to ","type":"text"},{"text":"prometheusScrape","type":"text","marks":[{"type":"code"}]},{"text":" to be enabled in the datadog values yaml, the pods needs to have the following annotations. Note that if the application pods are deployed using helm, typically the helm values support a ","type":"text"},{"text":"podAnnotations","type":"text","marks":[{"type":"code"}]},{"text":" section.","type":"text"},{"type":"hardBreak"},{"type":"hardBreak"}]},{"type":"codeBlock","content":[{"text":" prometheus.io/path: \n prometheus.io/port: \n prometheus.io/scrape: \"true\"","type":"text"}]},{"type":"heading","attrs":{"level":1},"content":[{"type":"hardBreak"},{"type":"hardBreak"},{"text":"Metadata for metrics collected using the openmetrics check","type":"text"}]},{"type":"paragraph","content":[{"text":"If using the above configuration, then the openmetrics check is enabled in the agent. See here for more details on what ","type":"text"},{"text":"openmetrics","type":"text","marks":[{"type":"link","attrs":{"href":"https://docs.datadoghq.com/containers/kubernetes/prometheus/?tab=kubernetesadv1"}}]},{"text":" check does. Kloudfuse agent (installed using the provided values file) employs a custom check (kf_openmetrics) to collect metadata (the “Description” and “Type” of metrics) for metrics collected using the openmetrics check, which. by default, doesn’t collect any metadata. To enable collection of metadata for these metrics, the sources are required to be annotated which will enable the agent ","type":"text"},{"text":"auto-discovery","type":"text","marks":[{"type":"link","attrs":{"href":"https://docs.datadoghq.com/containers/kubernetes/integrations/?tab=kubernetesadv2"}}]},{"text":" for these pods and execute the custom check. ","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Kubernetes environment","type":"text"}]},{"type":"paragraph","content":[{"text":"To enable metrics metadata to be collected from a Kubernetes pod (as shown in an ","type":"text","marks":[{"type":"annotation","attrs":{"annotationType":"inlineComment","id":"1e245ca8-75e5-498a-9025-04dc6e8c4e7d"}}]},{"text":"example here","type":"text","marks":[{"type":"annotation","attrs":{"annotationType":"inlineComment","id":"1e245ca8-75e5-498a-9025-04dc6e8c4e7d"}},{"type":"link","attrs":{"href":"https://docs.datadoghq.com/containers/kubernetes/integrations/?tab=kubernetesadv1#configuration"}}]},{"text":") follow these steps (Note that this can be done in helm as well for each of the deployment/sts):","type":"text","marks":[{"type":"annotation","attrs":{"annotationType":"inlineComment","id":"1e245ca8-75e5-498a-9025-04dc6e8c4e7d"}}]}]},{"type":"codeBlock","content":[{"text":"apiVersion: v1\nkind: Pod\n# (...)\nmetadata:\n name: ''\nannotations:\n ad.datadoghq.com/.check_names: '[\"kf_openmetrics\"]'\n ad.datadoghq.com/.init_configs: '[{}]'\n ad.datadoghq.com/.instances: '[ { \"openmetrics_endpoint\": \"\"}\n ]'\n # (...)\nspec:\n containers:\n - name: ''\n# (...)","type":"text"}]},{"type":"heading","attrs":{"level":1},"content":[{"text":"Advance monitoring using Knight","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Enable kubernetes_state_metrics","type":"text"}]},{"type":"paragraph","content":[{"text":"Advance monitoring","type":"text","marks":[{"type":"link","attrs":{"href":"https://kloudfuse.atlassian.net/wiki/spaces/EX/pages/756056089"}}]},{"text":" (Kfuse 1.3 or higher) currently has dependency on ","type":"text"},{"text":"kubernetes_state_metrics","type":"text","marks":[{"type":"link","attrs":{"href":"https://github.com/kubernetes/kube-state-metrics"}}]},{"text":" (KSM) check which is not enabled in the newer version of the agent (2.0) by default. Please ensure that the agent continues to capture these metrics through KSM. To do that, please add/update the dd-agent values file as follows: ","type":"text"}]},{"type":"codeBlock","content":[{"text":"datadog:\n kubeStateMetricsEnabled: true\n kubeStateMetricsCore:\n enabled: true\n ignoreLegacyKSMCheck: false","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Enable Knight based monitoring in kfuse","type":"text"}]},{"type":"paragraph","content":[{"text":"Add ","type":"text"},{"text":"knightEnabled","type":"text","marks":[{"type":"code"}]},{"text":" in the ","type":"text"},{"text":"custom-values.yaml","type":"text","marks":[{"type":"code"}]},{"text":" and then upgrade the cluster.","type":"text"}]},{"type":"codeBlock","content":[{"text":"ingester:\n ...\n config:\n knightEnabled: true","type":"text"}]}],"version":1}

Browser not supported