Merge pull request #515 from cisagov/improvement/fix-list-displays

Generate individual software list files instead of a single software list

.github/CODEOWNERS

@@ -5,6 +5,9 @@
 # requested for review when someone opens a pull request.
 * @iainDe @justmurphy @Lcerkov
 
+# These folks own the automated list update configuration.
+/config/ @mcdonnnj
+
 # These folks own any dot-files in the repository.
 .* @dav3r @felddy @jsf9k @mcdonnnj
 

.github/workflows/build.yml

@@ -15,23 +15,46 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
+      - id: setup-env
+        uses: cisagov/setup-env-github-action@develop
       - uses: actions/checkout@v2
       - id: setup-python
         uses: actions/setup-python@v2
         with:
           python-version: "3.10"
+      # We need the Go version and Go cache location for the actions/cache step,
+      # so the Go installation must happen before that.
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '1.16'
+      - name: Store installed Go version
+        id: go-version
+        run: |
+          echo "::set-output name=version::"\
+          "$(go version | sed 's/^go version go\([0-9.]\+\) .*/\1/')"
+      - name: Lookup Go cache directory
+        id: go-cache
+        run: |
+          echo "::set-output name=dir::$(go env GOCACHE)"
       - uses: actions/cache@v2
         env:
           BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\
-            py${{ steps.setup-python.outputs.python-version }}"
+            py${{ steps.setup-python.outputs.python-version }}-\
+            go${{ steps.go-version.outputs.version }}-"
         with:
           path: |
             ${{ env.PIP_CACHE_DIR }}
             ${{ env.PRE_COMMIT_CACHE_DIR }}
+            ${{ steps.go-cache.outputs.dir }}
           key: "${{ env.BASE_CACHE_KEY }}\
             ${{ hashFiles('**/.pre-commit-config.yaml') }}"
           restore-keys: |
             ${{ env.BASE_CACHE_KEY }}
+      - name: Install shfmt
+        env:
+          PACKAGE_URL: mvdan.cc/sh/v3/cmd/shfmt
+          PACKAGE_VERSION: ${{ steps.setup-env.outputs.shfmt-version }}
+        run: go install ${PACKAGE_URL}@${PACKAGE_VERSION}
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip setuptools wheel

.github/workflows/update_software_lists.yml

@@ -1,5 +1,5 @@
 ---
-name: Update the software list
+name: Update the software lists
 
 on:
   push:
@@ -8,7 +8,7 @@ on:
 
 env:
   PIP_CACHE_DIR: ~/.cache/pip
-  TESTING_BRANCH_BASE: testing/update_software_list
+  TESTING_BRANCH_BASE: testing/update_software_lists
 
 jobs:
   setup:
@@ -34,12 +34,12 @@ jobs:
         env:
           BASE_BRANCH: ${{ env.TESTING_BRANCH_BASE }}
           COMMIT_SHA: ${{ github.sha }}
-  generate_list_update:
+  generate_updates:
     runs-on: ubuntu-latest
     needs: setup
     outputs:
       # If changes are detected then a commit will have been pushed
-      updated_list: ${{ steps.commit-for-testing.outputs.changes_detected }}
+      has_updates: ${{ steps.commit-for-testing.outputs.changes_detected }}
     # Don't run if we're seeing an update push
     if: github.actor != needs.setup.outputs.git_user
     steps:
@@ -58,7 +58,7 @@ jobs:
           path: |
             ${{ env.PIP_CACHE_DIR }}
           key: "${{ env.BASE_CACHE_KEY }}\
-            ${{ hashFiles('.github/workflows/update_software_list.yml') }}-\
+            ${{ hashFiles('.github/workflows/update_software_lists.yml') }}-\
             ${{ hashFiles('config/requirements.txt') }}"
           restore-keys: |
             ${{ env.BASE_CACHE_KEY }}
@@ -68,35 +68,23 @@ jobs:
         run: pip install --upgrade --requirement config/requirements.txt
       - name: Create the branch for test validation
         run: git switch --create ${{ needs.setup.outputs.testing_branch }}
-      - name: Normalize individual cisagov_*.yml files
-        run: |
-          for file in data/cisagov_*yml; do \
-            normalize-yml --cisagov-format "$file" > "$file".tmp; \
-            mv --force "$file".tmp "$file"; \
-          done
-      - name: Update the comprehensive cisagov YAML file
-        run: normalize-yml --cisagov-format data/cisagov_*.yml > data/cisagov.yml
-      - name: Generate a normalized YAML file from all source YAML files
-        run: normalize-yml data/cisagov.yml > normalized.yml
-      - name: Generate a Markdown table from the normalized YAML file
-        run: yml2md normalized.yml > table_data.md
-      - name: Generate a new software list from the updated data
-        run: md-from-template config/SOFTWARE-LIST.tpl.md table_data.md > SOFTWARE-LIST.md
+      - name: Update the YAML and Markdown files as appropriate
+        run: config/update_software_lists.sh
       - id: commit-for-testing
         uses: stefanzweifel/git-auto-commit-action@v4
         with:
           branch: ${{ needs.setup.outputs.testing_branch }}
-          commit_message: Update the software list
+          commit_message: Normalize YAML files and update the software lists
           commit_user_name: ${{ needs.setup.outputs.git_user }}
           commit_user_email: ${{ needs.setup.outputs.git_email }}
           commit_author: ${{ needs.setup.outputs.git_author }}
-          file_pattern: SOFTWARE-LIST.md data/cisagov*.yml
-  merge_list_update:
+          file_pattern: data/cisagov*.yml software_lists/software_list_*.md
+  merge_updates:
     runs-on: ubuntu-latest
     needs:
       - setup
-      - generate_list_update
-    if: needs.generate_list_update.outputs.updated_list == 'true'
+      - generate_updates
+    if: needs.generate_updates.outputs.has_updates == 'true'
     steps:
       - uses: actions/checkout@v2
         with:

.pre-commit-config.yaml

@@ -46,3 +46,23 @@ repos:
       - id: yamllint
         args:
           - --strict
+
+  # Shell script hooks
+  - repo: https://github.com/cisagov/pre-commit-shfmt
+    rev: v0.0.2
+    hooks:
+      - id: shfmt
+        args:
+          # Indent by two spaces
+          - -i
+          - '2'
+          # Binary operators may start a line
+          - -bn
+          # Switch cases are indented
+          - -ci
+          # Redirect operators are followed by a space
+          - -sr
+  - repo: https://github.com/detailyang/pre-commit-shell
+    rev: 1.0.5
+    hooks:
+      - id: shell-lint

README.md

@@ -87,4 +87,4 @@ in [`PULL-EXAMPLE.md`](PULL-EXAMPLE.md).
 
 ## Software List ##
 
-To view the full list of vendors & software [click here](./SOFTWARE-LIST.md).
+To view the full list of vendors & software [click here](./software_lists/README.md).

config/SOFTWARE-LIST.tpl.md

@@ -1,5 +1,15 @@
 # CISA Log4j (CVE-2021-44228) Affected Vendor & Software List #
 
+[0-9](software_list_Non-Alphabet.md) [A](software_list_A.md) [B](software_list_B.md)
+[C](software_list_C.md) [D](software_list_D.md) [E](software_list_E.md)
+[F](software_list_F.md) [G](software_list_G.md) [H](software_list_H.md)
+[I](software_list_I.md) [J](software_list_J.md) [K](software_list_K.md)
+[L](software_list_L.md) [M](software_list_M.md) [N](software_list_N.md)
+[O](software_list_O.md) [P](software_list_P.md) [Q](software_list_Q.md)
+[R](software_list_R.md) [S](software_list_S.md) [T](software_list_T.md)
+[U](software_list_U.md) [V](software_list_V.md) [W](software_list_W.md)
+[X](software_list_X.md) [Y](software_list_Y.md) [Z](software_list_Z.md)
+
 ## Status Descriptions ##
 
 | Status | Description |

config/update_software_lists.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+# This script is used to do the following:
+# - Normalize each of the data/cisagov_*.yml files.
+# - Update the comprehensive data/cisagov.yml file.
+# - Generate each software_lists/software_list_*.md file from its respective
+#   data/cisagov_*.yml file.
+
+set -o nounset
+set -o errexit
+set -o pipefail
+
+OUTPUT_DIRECTORY=software_lists
+TEMPLATE_FILE=config/SOFTWARE-LIST.tpl.md
+
+echo Normalize individual cisagov YAML files
+for file in data/cisagov_*.yml; do
+  echo "  $file..."
+  normalize-yml --cisagov-format "$file" > "$file".tmp
+  mv --force "$file".tmp "$file"
+done
+
+echo Update the comprehensive cisagov YAML file
+normalize-yml --cisagov-format data/cisagov_*.yml > data/cisagov.yml
+
+echo Generate Markdown files from the individual cisagov YAML files
+for file in data/cisagov_*.yml; do
+  echo "  $file..."
+  # Convert the file path data/cisagov_*.yml to software_list_*.md
+  md_file=$(echo "$file" | sed 's/data\/cisagov_\(.\+\)yml/software_list_\1md/g')
+  normalize-yml "$file" > "$file.tmp"
+  yml2md "$file.tmp" > "$md_file.tmp"
+  md-from-template $TEMPLATE_FILE "$md_file.tmp" > "$OUTPUT_DIRECTORY/$md_file"
+done

software_lists/README.md

@@ -0,0 +1,25 @@
+# CISA Log4j (CVE-2021-44228) Affected Vendor & Software Lists #
+
+## Contributing Guidelines ##
+
+We welcome contributions!  Please see [`CONTRIBUTING.md`](../CONTRIBUTING.md) for
+details.
+
+### Creating a pull request ###
+
+Instructions for creating a pull request using the GitHub Web UI can be found
+in [`PULL-EXAMPLE.md`](../PULL-EXAMPLE.md).
+
+## Software Lists ##
+
+To view the full lists of vendors & software click below:
+
+[0-9](software_list_Non-Alphabet.md) [A](software_list_A.md) [B](software_list_B.md)
+[C](software_list_C.md) [D](software_list_D.md) [E](software_list_E.md)
+[F](software_list_F.md) [G](software_list_G.md) [H](software_list_H.md)
+[I](software_list_I.md) [J](software_list_J.md) [K](software_list_K.md)
+[L](software_list_L.md) [M](software_list_M.md) [N](software_list_N.md)
+[O](software_list_O.md) [P](software_list_P.md) [Q](software_list_Q.md)
+[R](software_list_R.md) [S](software_list_S.md) [T](software_list_T.md)
+[U](software_list_U.md) [V](software_list_V.md) [W](software_list_W.md)
+[X](software_list_X.md) [Y](software_list_Y.md) [Z](software_list_Z.md)