Update software list generation

Consolidate all update tasks into a single bash script that is run by
the GitHub Actions workflow. This also switches to generating
individual Markdown files for each data/cisagov_*.yml file.
Nicholas McDonnell 2 years ago
parent d5345e33cf
commit bc0e017b62
No known key found for this signature in database
GPG Key ID: 7994ADE2A56BE5D1
  1. 20
  2. 34

@ -68,29 +68,17 @@ jobs:
run: pip install --upgrade --requirement config/requirements.txt
- name: Create the branch for test validation
run: git switch --create ${{ needs.setup.outputs.testing_branch }}
- name: Normalize individual cisagov_*.yml files
run: |
for file in data/cisagov_*yml; do \
normalize-yml --cisagov-format "$file" > "$file".tmp; \
mv --force "$file".tmp "$file"; \
- name: Update the comprehensive cisagov YAML file
run: normalize-yml --cisagov-format data/cisagov_*.yml > data/cisagov.yml
- name: Generate a normalized YAML file from all source YAML files
run: normalize-yml data/cisagov.yml > normalized.yml
- name: Generate a Markdown table from the normalized YAML file
run: yml2md normalized.yml > table_data.md
- name: Generate a new software list from the updated data
run: md-from-template config/SOFTWARE-LIST.tpl.md table_data.md > SOFTWARE-LIST.md
- name: Update the YAML and Markdown files as appropriate
run: config/update_software_lists.sh
- id: commit-for-testing
uses: stefanzweifel/git-auto-commit-action@v4
branch: ${{ needs.setup.outputs.testing_branch }}
commit_message: Update the software list
commit_message: Normalize YAML files and update the software lists
commit_user_name: ${{ needs.setup.outputs.git_user }}
commit_user_email: ${{ needs.setup.outputs.git_email }}
commit_author: ${{ needs.setup.outputs.git_author }}
file_pattern: SOFTWARE-LIST.md data/cisagov*.yml
file_pattern: data/cisagov*.yml software_lists/software_list_*.md
runs-on: ubuntu-latest

@ -0,0 +1,34 @@
#!/usr/bin/env bash
# This script is used to do the following:
# - Normalize each of the data/cisagov_*.yml files.
# - Update the comprehensive data/cisagov.yml file.
# - Generate each software_lists/software_list_*.md file from its respective
# data/cisagov_*.yml file.
set -o nounset
set -o errexit
set -o pipefail
echo Normalize individual cisagov YAML files
for file in data/cisagov_*.yml; do
echo " $file..."
normalize-yml --cisagov-format "$file" > "$file".tmp
mv --force "$file".tmp "$file"
echo Update the comprehensive cisagov YAML file
normalize-yml --cisagov-format data/cisagov_*.yml > data/cisagov.yml
echo Generate Markdown files from the individual cisagov YAML files
for file in data/cisagov_*.yml; do
echo " $file..."
# Convert the file path data/cisagov_*.yml to software_list_*.md
md_file=$(echo "$file" | sed 's/data\/cisagov_\(.\+\)yml/software_list_\1md/g')
normalize-yml "$file" > "$file.tmp"
yml2md "$file.tmp" > "$md_file.tmp"
md-from-template $TEMPLATE_FILE "$md_file.tmp" > "$OUTPUT_DIRECTORY/$md_file"