import json import re import shutil from pathlib import Path # Dictionary responsible for making the symbolic links at the end of the script's run. symlink_dict = {} def cleanup(): """ clean directories that are either data collection or exporting integrations """ for element in Path("src/go/plugin/go.d/modules").glob('**/*/'): if "integrations" in str(element): shutil.rmtree(element) for element in Path("src/collectors").glob('**/*/'): # print(element) if "integrations" in str(element): shutil.rmtree(element) for element in Path("src/exporting").glob('**/*/'): if "integrations" in str(element): shutil.rmtree(element) for element in Path("integrations/cloud-notifications").glob('**/*/'): if "integrations" in str(element) and not "metadata.yaml" in str(element): shutil.rmtree(element) for element in Path("integrations/logs").glob('**/*/'): if "integrations" in str(element) and "metadata.yaml" not in str(element): shutil.rmtree(element) for element in Path("integrations/cloud-authentication").glob('**/*/'): if "integrations" in str(element) and not "metadata.yaml" in str(element): shutil.rmtree(element) def generate_category_from_name(category_fragment, category_array): """ Takes a category ID in splitted form ("." as delimiter) and the array of the categories, and returns the proper category name that Learn expects. """ category_name = "" i = 0 dummy_id = category_fragment[0] while i < len(category_fragment): for category in category_array: if dummy_id == category['id']: category_name = category_name + "/" + category["name"] try: # print("equals") # print(fragment, category_fragment[i+1]) dummy_id = dummy_id + "." + category_fragment[i + 1] # print(dummy_id) except IndexError: return category_name.split("/", 1)[1] category_array = category['children'] break i += 1 def clean_and_write(md, path): """ This function takes care of the special details element, and converts it to the equivalent that md expects. Then it writes the buffer on the file provided. """ # clean first, replace md = md.replace("{% details summary=\"", "
") md = md.replace("{% details open=true summary=\"", "
") md = md.replace("\" %}", "\n") md = md.replace("{% /details %}", "
\n") path.write_text(md) def add_custom_edit_url(markdown_string, meta_yaml_link, sidebar_label_string, mode='default'): """ Takes a markdown string and adds a "custom_edit_url" metadata to the metadata field """ output = "" path_to_md_file = "" if mode == 'default': path_to_md_file = f'{meta_yaml_link.replace("/metadata.yaml", "")}/integrations/{clean_string(sidebar_label_string)}' elif mode == 'cloud-notification': path_to_md_file = meta_yaml_link.replace("metadata.yaml", f'integrations/{clean_string(sidebar_label_string)}') elif mode == 'agent-notification': path_to_md_file = meta_yaml_link.replace("metadata.yaml", "README") elif mode == 'cloud-authentication': path_to_md_file = meta_yaml_link.replace("metadata.yaml", f'integrations/{clean_string(sidebar_label_string)}') elif mode == 'logs': path_to_md_file = meta_yaml_link.replace("metadata.yaml", f'integrations/{clean_string(sidebar_label_string)}') output = markdown_string.replace( " {create_overview(integration, integration['meta']['monitored_instance']['icon_filename'])}""" if integration['metrics']: md += f""" {integration['metrics']} """ if integration['alerts']: md += f""" {integration['alerts']} """ if integration['setup']: md += f""" {integration['setup']} """ if integration['troubleshooting']: md += f""" {integration['troubleshooting']} """ except Exception as e: print("Exception in collector md construction", e, integration['id']) # EXPORTERS elif mode == 'exporter': try: # initiate the variables for the exporter meta_yaml = integration['edit_link'].replace("blob", "edit") sidebar_label = integration['meta']['name'] learn_rel_path = generate_category_from_name(integration['meta']['categories'][0].split("."), categories) # build the markdown string md = \ f""" {create_overview(integration, integration['meta']['icon_filename'])}""" if integration['setup']: md += f""" {integration['setup']} """ if integration['troubleshooting']: md += f""" {integration['troubleshooting']} """ except Exception as e: print("Exception in exporter md construction", e, integration['id']) # NOTIFICATIONS elif mode == 'agent-notification': try: # initiate the variables for the notification method meta_yaml = integration['edit_link'].replace("blob", "edit") sidebar_label = integration['meta']['name'] learn_rel_path = generate_category_from_name(integration['meta']['categories'][0].split("."), categories) # build the markdown string md = \ f""" {create_overview(integration, integration['meta']['icon_filename'], "overview")}""" if integration['setup']: md += f""" {integration['setup']} """ if integration['troubleshooting']: md += f""" {integration['troubleshooting']} """ except Exception as e: print("Exception in notification md construction", e, integration['id']) elif mode == 'cloud-notification': try: # initiate the variables for the notification method meta_yaml = integration['edit_link'].replace("blob", "edit") sidebar_label = integration['meta']['name'] learn_rel_path = generate_category_from_name(integration['meta']['categories'][0].split("."), categories) # build the markdown string md = \ f""" {create_overview(integration, integration['meta']['icon_filename'], "")}""" if integration['setup']: md += f""" {integration['setup']} """ if integration['troubleshooting']: md += f""" {integration['troubleshooting']} """ except Exception as e: print("Exception in notification md construction", e, integration['id']) elif mode == 'logs': try: # initiate the variables for the logs integration meta_yaml = integration['edit_link'].replace("blob", "edit") sidebar_label = integration['meta']['name'] learn_rel_path = generate_category_from_name(integration['meta']['categories'][0].split("."), categories) # build the markdown string md = \ f""" {create_overview(integration, integration['meta']['icon_filename'])}""" if integration['setup']: md += f""" {integration['setup']} """ except Exception as e: print("Exception in logs md construction", e, integration['id']) # AUTHENTICATIONS elif mode == 'authentication': if True: # initiate the variables for the authentication method meta_yaml = integration['edit_link'].replace("blob", "edit") sidebar_label = integration['meta']['name'] learn_rel_path = generate_category_from_name(integration['meta']['categories'][0].split("."), categories) # build the markdown string md = \ f""" {create_overview(integration, integration['meta']['icon_filename'])}""" if integration['setup']: md += f""" {integration['setup']} """ if integration['troubleshooting']: md += f""" {integration['troubleshooting']} """ # except Exception as e: # print("Exception in authentication md construction", e, integration['id']) if "community" in integration['meta'].keys(): community = "" else: community = "" return meta_yaml, sidebar_label, learn_rel_path, md, community def build_path(meta_yaml_link): """ function that takes a metadata yaml file link, and makes it into a path that gets used to write to a file. """ return meta_yaml_link.replace("https://github.com/netdata/", "") \ .split("/", 1)[1] \ .replace("edit/master/", "") \ .replace("/metadata.yaml", "") def write_to_file(path, md, meta_yaml, sidebar_label, community, mode='default'): """ takes the arguments needed to write the integration markdown to the proper file. """ upper, lower = md.split("##", 1) md = upper + community + f"\n\n##{lower}" if mode == 'default': # Only if the path exists, this caters for running the same script on both the go and netdata repos. if Path(path).exists(): if not Path(f'{path}/integrations').exists(): Path(f'{path}/integrations').mkdir() try: md = add_custom_edit_url(md, meta_yaml, sidebar_label) clean_and_write( md, Path(f'{path}/integrations/{clean_string(sidebar_label)}.md') ) except FileNotFoundError as e: print("Exception in writing to file", e) # If we only created one file inside the directory, add the entry to the symlink_dict, so we can make the symbolic link if len(list(Path(f'{path}/integrations').iterdir())) == 1: symlink_dict.update( {path: f'integrations/{clean_string(sidebar_label)}.md'}) else: try: symlink_dict.pop(path) except KeyError: # We don't need to print something here. pass elif mode == 'cloud-notification': # for cloud notifications we generate them near their metadata.yaml name = clean_string(integration['meta']['name']) if not Path(f'{path}/integrations').exists(): Path(f'{path}/integrations').mkdir() # proper_edit_name = meta_yaml.replace( # "metadata.yaml", f'integrations/{clean_string(sidebar_label)}.md\"') md = add_custom_edit_url(md, meta_yaml, sidebar_label, mode='cloud-notification') finalpath = f'{path}/integrations/{name}.md' try: clean_and_write( md, Path(finalpath) ) except FileNotFoundError as e: print("Exception in writing to file", e) elif mode == 'agent-notification': # add custom_edit_url as the md file, so we can have uniqueness in the ingest script # afterwards the ingest will replace this metadata with meta_yaml md = add_custom_edit_url(md, meta_yaml, sidebar_label, mode='agent-notification') finalpath = f'{path}/README.md' try: clean_and_write( md, Path(finalpath) ) except FileNotFoundError as e: print("Exception in writing to file", e) elif mode == 'logs': # for logs we generate them near their metadata.yaml name = clean_string(integration['meta']['name']) if not Path(f'{path}/integrations').exists(): Path(f'{path}/integrations').mkdir() # proper_edit_name = meta_yaml.replace( # "metadata.yaml", f'integrations/{clean_string(sidebar_label)}.md\"') md = add_custom_edit_url(md, meta_yaml, sidebar_label, mode='logs') finalpath = f'{path}/integrations/{name}.md' try: clean_and_write( md, Path(finalpath) ) except FileNotFoundError as e: print("Exception in writing to file", e) elif mode == 'authentication': name = clean_string(integration['meta']['name']) if not Path(f'{path}/integrations').exists(): Path(f'{path}/integrations').mkdir() # proper_edit_name = meta_yaml.replace( # "metadata.yaml", f'integrations/{clean_string(sidebar_label)}.md\"') md = add_custom_edit_url(md, meta_yaml, sidebar_label, mode='cloud-authentication') finalpath = f'{path}/integrations/{name}.md' try: clean_and_write( md, Path(finalpath) ) except FileNotFoundError as e: print("Exception in writing to file", e) def make_symlinks(symlink_dict): """ takes a dictionary with directories that have a 1:1 relationship between their README and the integration (only one) inside the "integrations" folder. """ for element in symlink_dict: if not Path(f'{element}/README.md').exists(): Path(f'{element}/README.md').touch() try: # Remove the README to prevent it being a normal file Path(f'{element}/README.md').unlink() except FileNotFoundError: continue # and then make a symlink to the actual markdown Path(f'{element}/README.md').symlink_to(symlink_dict[element]) filepath = Path(f'{element}/{symlink_dict[element]}') md = filepath.read_text() # This preserves the custom_edit_url for most files as it was, # so the existing links don't break, this is vital for link replacement afterwards filepath.write_text(md.replace( f'{element}/{symlink_dict[element]}', f'{element}/README.md')) cleanup() categories, integrations = read_integrations_js('integrations/integrations.js') # Iterate through every integration for integration in integrations: if integration['integration_type'] == "collector": meta_yaml, sidebar_label, learn_rel_path, md, community = build_readme_from_integration( integration, mode='collector') path = build_path(meta_yaml) write_to_file(path, md, meta_yaml, sidebar_label, community) else: # kind of specific if clause, so we can avoid running excessive code in the go repo if integration['integration_type'] == "exporter": meta_yaml, sidebar_label, learn_rel_path, md, community = build_readme_from_integration( integration, mode='exporter') path = build_path(meta_yaml) write_to_file(path, md, meta_yaml, sidebar_label, community) elif integration['integration_type'] == "agent_notification": meta_yaml, sidebar_label, learn_rel_path, md, community = build_readme_from_integration( integration, mode='agent-notification') path = build_path(meta_yaml) write_to_file(path, md, meta_yaml, sidebar_label, community, mode='agent-notification') elif integration['integration_type'] == "cloud_notification": meta_yaml, sidebar_label, learn_rel_path, md, community = build_readme_from_integration( integration, mode='cloud-notification') path = build_path(meta_yaml) write_to_file(path, md, meta_yaml, sidebar_label, community, mode='cloud-notification') elif integration['integration_type'] == "logs": meta_yaml, sidebar_label, learn_rel_path, md, community = build_readme_from_integration( integration, mode='logs') path = build_path(meta_yaml) write_to_file(path, md, meta_yaml, sidebar_label, community, mode='logs') elif integration['integration_type'] == "authentication": meta_yaml, sidebar_label, learn_rel_path, md, community = build_readme_from_integration( integration, mode='authentication') path = build_path(meta_yaml) write_to_file(path, md, meta_yaml, sidebar_label, community, mode='authentication') make_symlinks(symlink_dict)