Add integrations JSON file for website usage. (#15959)

* Add generation of JSON integrations data.

* Clean up generated data for JSON file.

* Properly clean up deploy entry info.

* Fix argument order for regex substitutions.

integrations/deploy.yaml

@@ -35,6 +35,8 @@
             --stable-channel{% if $showClaimingOptions %} --claim-token {% claim_token %} --claim-rooms {% $claim_rooms %} --claim-url {% claim_url %}{% /if %}
   additional_info: &ref_containers >
     Did you know you can also deploy Netdata on your OS using {% goToCategory navigateToSettings=$navigateToSettings categoryId="deploy.docker-kubernetes" %}Kubernetes{% /goToCategory %} or {% goToCategory categoryId="deploy.docker-kubernetes" %}Docker{% /goToCategory %}?
+  clean_additional_info: &ref_clean_containers >
+    Did you know you can also deploy Netdata on your OS using Kubernetes or Docker?
   related_resources: {}
   platform_info:
     group: ''
@@ -85,7 +87,7 @@
     icon_filename: 'rhel.png'
   most_popular: false
   platform_info:
-    group: 'include'
+    group: 'no_include'
     distro: 'rhel'
   quick_start: -1
 - <<: *linux
@@ -148,6 +150,18 @@
     group: 'include'
     distro: 'centos'
   quick_start: -1
+- <<: *linux
+  id: deploy-centos-stream
+  meta:
+    <<: *linux_meta
+    name: CentOS Stream
+    link: https://www.centos.org/centos-stream
+    icon_filename: 'centos.png'
+  most_popular: false
+  platform_info:
+    group: 'include'
+    distro: 'centos-stream'
+  quick_start: -1
 - <<: *linux
   id: deploy-manjarolinux
   meta:
@@ -200,9 +214,10 @@
   methods:
     - *ks_curl
   additional_info: *ref_containers
+  clean_additional_info: *ref_clean_containers
   related_resources: {}
   platform_info:
-    group: 'include'
+    group: 'no_include'
     distro: 'macos'
   quick_start: 5
 - id: deploy-docker
@@ -573,6 +588,6 @@
     Netdata can also be installed via [FreeBSD ports](https://www.freshports.org/net-mgmt/netdata).
   related_resources: {}
   platform_info:
-    group: 'include'
+    group: 'no_include'
     distro: 'freebsd'
   quick_start: 6

integrations/gen_integrations.py

@@ -2,8 +2,10 @@
 
 import json
 import os
+import re
 import sys
 
+from copy import deepcopy
 from pathlib import Path
 
 from jsonschema import Draft7Validator, ValidationError
@@ -17,6 +19,7 @@ GO_REPO = 'netdata/go.d.plugin'
 INTEGRATIONS_PATH = Path(__file__).parent
 TEMPLATE_PATH = INTEGRATIONS_PATH / 'templates'
 OUTPUT_PATH = INTEGRATIONS_PATH / 'integrations.js'
+JSON_PATH = INTEGRATIONS_PATH / 'integrations.json'
 CATEGORIES_FILE = INTEGRATIONS_PATH / 'categories.yaml'
 REPO_PATH = INTEGRATIONS_PATH.parent
 SCHEMA_PATH = INTEGRATIONS_PATH / 'schemas'
@@ -65,6 +68,9 @@ NOTIFICATION_RENDER_KEYS = [
     'troubleshooting',
 ]
 
+CUSTOM_TAG_PATTERN = re.compile('\\{% if .*?%\\}.*?\\{% /if %\\}|\\{%.*?%\\}', flags=re.DOTALL)
+FIXUP_BLANK_PATTERN = re.compile('\\\\\\n *\\n')
+
 GITHUB_ACTIONS = os.environ.get('GITHUB_ACTIONS', False)
 DEBUG = os.environ.get('DEBUG', False)
 
@@ -438,12 +444,17 @@ def render_collectors(categories, collectors, ids):
     debug('Removing duplicate collectors.')
 
     collectors, ids = dedupe_integrations(collectors, ids)
+    clean_collectors = []
 
     idmap = {i['id']: i for i in collectors}
 
     for item in collectors:
         debug(f'Processing { item["id"] }.')
 
+        item['edit_link'] = make_edit_link(item)
+
+        clean_item = deepcopy(item)
+
         related = []
 
         for res in item['meta']['related_resources']['integrations']['list']:
@@ -487,23 +498,27 @@ def render_collectors(categories, collectors, ids):
         for key in COLLECTOR_RENDER_KEYS:
             if key in item.keys():
                 template = get_jinja_env().get_template(f'{ key }.md')
-                data = template.render(entry=item, related=related)
+                data = template.render(entry=item, related=related, clean=False)
+                clean_data = template.render(entry=item, related=related, clean=True)
 
                 if 'variables' in item['meta']['monitored_instance']:
                     template = get_jinja_env().from_string(data)
                     data = template.render(variables=item['meta']['monitored_instance']['variables'])
+                    template = get_jinja_env().from_string(clean_data)
+                    clean_data = template.render(variables=item['meta']['monitored_instance']['variables'])
             else:
                 data = ''
+                clean_data = ''
 
             item[key] = data
+            clean_item[key] = clean_data
 
-        item['edit_link'] = make_edit_link(item)
+        for k in ['_src_path', '_repo', '_index']:
+            del item[k], clean_item[k]
 
-        del item['_src_path']
-        del item['_repo']
-        del item['_index']
+        clean_collectors.append(clean_item)
 
-    return collectors, ids
+    return collectors, clean_collectors, ids
 
 
 def render_deploy(distros, categories, deploy, ids):
@@ -514,11 +529,14 @@ def render_deploy(distros, categories, deploy, ids):
     debug('Checking deployment ids.')
 
     deploy, ids = dedupe_integrations(deploy, ids)
+    clean_deploy = []
 
     template = get_jinja_env().get_template('platform_info.md')
 
     for item in deploy:
         debug(f'Processing { item["id"] }.')
+        item['edit_link'] = make_edit_link(item)
+        clean_item = deepcopy(item)
 
         if item['platform_info']['group']:
             entries = [
@@ -532,16 +550,27 @@ def render_deploy(distros, categories, deploy, ids):
         else:
             entries = []
 
-        data = template.render(entries=entries)
+        data = template.render(entries=entries, clean=False)
+        clean_data = template.render(entries=entries, clean=True)
+
+        for method in clean_item['methods']:
+            for command in method['commands']:
+                command['command'] = CUSTOM_TAG_PATTERN.sub('', command['command'])
+                command['command'] = FIXUP_BLANK_PATTERN.sub('', command['command'])
 
         item['platform_info'] = data
-        item['edit_link'] = make_edit_link(item)
+        clean_item['platform_info'] = clean_data
+
+        if 'clean_additional_info' in item:
+            clean_item['additional_info'] = item['clean_additional_info']
+            del item['clean_additional_info'], clean_item['clean_additional_info']
 
-        del item['_src_path']
-        del item['_repo']
-        del item['_index']
+        for k in ['_src_path', '_repo', '_index']:
+            del item[k], clean_item[k]
 
-    return deploy, ids
+        clean_deploy.append(clean_item)
+
+    return deploy, clean_deploy, ids
 
 
 def render_exporters(categories, exporters, ids):
@@ -553,27 +582,37 @@ def render_exporters(categories, exporters, ids):
 
     exporters, ids = dedupe_integrations(exporters, ids)
 
+    clean_exporters = []
+
     for item in exporters:
+        item['edit_link'] = make_edit_link(item)
+
+        clean_item = deepcopy(item)
+
         for key in EXPORTER_RENDER_KEYS:
             if key in item.keys():
                 template = get_jinja_env().get_template(f'{ key }.md')
-                data = template.render(entry=item)
+                data = template.render(entry=item, clean=False)
+                clean_data = template.render(entry=item, clean=True)
 
                 if 'variables' in item['meta']:
                     template = get_jinja_env().from_string(data)
-                    data = template.render(variables=item['meta']['variables'])
+                    data = template.render(variables=item['meta']['variables'], clean=False)
+                    template = get_jinja_env().from_string(clean_data)
+                    clean_data = template.render(variables=item['meta']['variables'], clean=True)
             else:
                 data = ''
+                clean_data = ''
 
             item[key] = data
+            clean_item[key] = clean_data
 
-        item['edit_link'] = make_edit_link(item)
+        for k in ['_src_path', '_repo', '_index']:
+            del item[k], clean_item[k]
 
-        del item['_src_path']
-        del item['_repo']
-        del item['_index']
+        clean_exporters.append(clean_item)
 
-    return exporters, ids
+    return exporters, clean_exporters, ids
 
 
 def render_notifications(categories, notifications, ids):
@@ -585,27 +624,37 @@ def render_notifications(categories, notifications, ids):
 
     notifications, ids = dedupe_integrations(notifications, ids)
 
+    clean_notifications = []
+
     for item in notifications:
+        item['edit_link'] = make_edit_link(item)
+
+        clean_item = deepcopy(item)
+
         for key in NOTIFICATION_RENDER_KEYS:
             if key in item.keys():
                 template = get_jinja_env().get_template(f'{ key }.md')
-                data = template.render(entry=item)
+                data = template.render(entry=item, clean=False)
+                clean_data = template.render(entry=item, clean=True)
 
                 if 'variables' in item['meta']:
                     template = get_jinja_env().from_string(data)
-                    data = template.render(variables=item['meta']['variables'])
+                    data = template.render(variables=item['meta']['variables'], clean=False)
+                    template = get_jinja_env().from_string(clean_data)
+                    clean_data = template.render(variables=item['meta']['variables'], clean=True)
             else:
                 data = ''
+                clean_data = ''
 
             item[key] = data
+            clean_item[key] = clean_data
 
-        item['edit_link'] = make_edit_link(item)
+        for k in ['_src_path', '_repo', '_index']:
+            del item[k], clean_item[k]
 
-        del item['_src_path']
-        del item['_repo']
-        del item['_index']
+        clean_notifications.append(clean_item)
 
-    return notifications, ids
+    return notifications, clean_notifications, ids
 
 
 def render_integrations(categories, integrations):
@@ -617,6 +666,13 @@ def render_integrations(categories, integrations):
     OUTPUT_PATH.write_text(data)
 
 
+def render_json(categories, integrations):
+    JSON_PATH.write_text(json.dumps({
+        'categories': categories,
+        'integrations': integrations,
+    }))
+
+
 def main():
     categories = load_categories()
     distros = load_yaml(DISTROS_FILE)
@@ -625,14 +681,17 @@ def main():
     exporters = load_exporters()
     notifications = load_notifications()
 
-    collectors, ids = render_collectors(categories, collectors, dict())
-    deploy, ids = render_deploy(distros, categories, deploy, ids)
-    exporters, ids = render_exporters(categories, exporters, ids)
-    notifications, ids = render_notifications(categories, notifications, ids)
+    collectors, clean_collectors, ids = render_collectors(categories, collectors, dict())
+    deploy, clean_deploy, ids = render_deploy(distros, categories, deploy, ids)
+    exporters, clean_exporters, ids = render_exporters(categories, exporters, ids)
+    notifications, clean_notifications, ids = render_notifications(categories, notifications, ids)
 
     integrations = collectors + deploy + exporters + notifications
     render_integrations(categories, integrations)
 
+    clean_integrations = clean_collectors + clean_deploy + clean_exporters + clean_notifications
+    render_json(categories, clean_integrations)
+
 
 if __name__ == '__main__':
     sys.exit(main())

integrations/schemas/deploy.json

@@ -68,6 +68,10 @@
         "type": "string",
         "description": "Any additional information about this platform."
       },
+      "clean_additional_info": {
+        "type": "string",
+        "description": "Any additional information about this platform, without any embedded custom tags."
+      },
       "related_resources": {
         "type": "object",
         "description": "TBD"

integrations/templates/metrics.md

@@ -1,7 +1,7 @@
 [% if entry.metrics.scopes %]
 ## Metrics
 
-[% if entry.metrics.folding.enabled %]
+[% if entry.metrics.folding.enabled and not clean %]
 {% details summary="[[ entry.metrics.folding.title ]]" %}
 [% endif %]
 Metrics grouped by *scope*.
@@ -39,7 +39,7 @@ Metrics:
 [% endfor %]
 
 [% endfor %]
-[% if entry.metrics.folding.enabled %]
+[% if entry.metrics.folding.enabled and not clean %]
 {% /details %}
 [% endif %]
 [% else %]

integrations/templates/setup.md

@@ -59,7 +59,7 @@ There is no configuration file.
 [[ entry.setup.configuration.options.description ]]
 
 [% if entry.setup.configuration.options.list %]
-[% if entry.setup.configuration.options.folding.enabled %]
+[% if entry.setup.configuration.options.folding.enabled and not clean %]
 {% details summary="[[ entry.setup.configuration.options.folding.title ]]" %}
 [% endif %]
 | Name | Description | Default | Required |
@@ -76,7 +76,7 @@ There is no configuration file.
 
 [% endif %]
 [% endfor %]
-[% if entry.setup.configuration.options.folding.enabled %]
+[% if entry.setup.configuration.options.folding.enabled and not clean %]
 {% /details %}
 [% endif %]
 [% elif not entry.setup.configuration.options.description %]
@@ -91,13 +91,13 @@ There are no configuration options.
 
 [[ example.description ]]
 
-[% if example.folding.enabled %]
+[% if example.folding.enabled and not clean %]
 {% details summary="[[ entry.setup.configuration.examples.folding.title ]]" %}
 [% endif %]
 ```yaml
 [[ example.config ]]
 ```
-[% if example.folding.enabled %]
+[% if example.folding.enabled and not clean %]
 {% /details %}
 [% endif %]
 [% endfor %]