Merge pull request #515 from cisagov/improvement/fix-list-displays

Generate individual software list files instead of a single software list
pull/513/head^2
Nick 2 years ago committed by GitHub
commit daddbc208b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 3
      .github/CODEOWNERS
  2. 25
      .github/workflows/build.yml
  3. 36
      .github/workflows/update_software_lists.yml
  4. 20
      .pre-commit-config.yaml
  5. 2
      README.md
  6. 3928
      SOFTWARE-LIST.md
  7. 10
      config/SOFTWARE-LIST.tpl.md
  8. 34
      config/update_software_lists.sh
  9. 25
      software_lists/README.md

@ -5,6 +5,9 @@
# requested for review when someone opens a pull request.
* @iainDe @justmurphy @Lcerkov
# These folks own the automated list update configuration.
/config/ @mcdonnnj
# These folks own any dot-files in the repository.
.* @dav3r @felddy @jsf9k @mcdonnnj

@ -15,23 +15,46 @@ jobs:
lint:
runs-on: ubuntu-latest
steps:
- id: setup-env
uses: cisagov/setup-env-github-action@develop
- uses: actions/checkout@v2
- id: setup-python
uses: actions/setup-python@v2
with:
python-version: "3.10"
# We need the Go version and Go cache location for the actions/cache step,
# so the Go installation must happen before that.
- uses: actions/setup-go@v2
with:
go-version: '1.16'
- name: Store installed Go version
id: go-version
run: |
echo "::set-output name=version::"\
"$(go version | sed 's/^go version go\([0-9.]\+\) .*/\1/')"
- name: Lookup Go cache directory
id: go-cache
run: |
echo "::set-output name=dir::$(go env GOCACHE)"
- uses: actions/cache@v2
env:
BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\
py${{ steps.setup-python.outputs.python-version }}"
py${{ steps.setup-python.outputs.python-version }}-\
go${{ steps.go-version.outputs.version }}-"
with:
path: |
${{ env.PIP_CACHE_DIR }}
${{ env.PRE_COMMIT_CACHE_DIR }}
${{ steps.go-cache.outputs.dir }}
key: "${{ env.BASE_CACHE_KEY }}\
${{ hashFiles('**/.pre-commit-config.yaml') }}"
restore-keys: |
${{ env.BASE_CACHE_KEY }}
- name: Install shfmt
env:
PACKAGE_URL: mvdan.cc/sh/v3/cmd/shfmt
PACKAGE_VERSION: ${{ steps.setup-env.outputs.shfmt-version }}
run: go install ${PACKAGE_URL}@${PACKAGE_VERSION}
- name: Install dependencies
run: |
python -m pip install --upgrade pip setuptools wheel

@ -1,5 +1,5 @@
---
name: Update the software list
name: Update the software lists
on:
push:
@ -8,7 +8,7 @@ on:
env:
PIP_CACHE_DIR: ~/.cache/pip
TESTING_BRANCH_BASE: testing/update_software_list
TESTING_BRANCH_BASE: testing/update_software_lists
jobs:
setup:
@ -34,12 +34,12 @@ jobs:
env:
BASE_BRANCH: ${{ env.TESTING_BRANCH_BASE }}
COMMIT_SHA: ${{ github.sha }}
generate_list_update:
generate_updates:
runs-on: ubuntu-latest
needs: setup
outputs:
# If changes are detected then a commit will have been pushed
updated_list: ${{ steps.commit-for-testing.outputs.changes_detected }}
has_updates: ${{ steps.commit-for-testing.outputs.changes_detected }}
# Don't run if we're seeing an update push
if: github.actor != needs.setup.outputs.git_user
steps:
@ -58,7 +58,7 @@ jobs:
path: |
${{ env.PIP_CACHE_DIR }}
key: "${{ env.BASE_CACHE_KEY }}\
${{ hashFiles('.github/workflows/update_software_list.yml') }}-\
${{ hashFiles('.github/workflows/update_software_lists.yml') }}-\
${{ hashFiles('config/requirements.txt') }}"
restore-keys: |
${{ env.BASE_CACHE_KEY }}
@ -68,35 +68,23 @@ jobs:
run: pip install --upgrade --requirement config/requirements.txt
- name: Create the branch for test validation
run: git switch --create ${{ needs.setup.outputs.testing_branch }}
- name: Normalize individual cisagov_*.yml files
run: |
for file in data/cisagov_*yml; do \
normalize-yml --cisagov-format "$file" > "$file".tmp; \
mv --force "$file".tmp "$file"; \
done
- name: Update the comprehensive cisagov YAML file
run: normalize-yml --cisagov-format data/cisagov_*.yml > data/cisagov.yml
- name: Generate a normalized YAML file from all source YAML files
run: normalize-yml data/cisagov.yml > normalized.yml
- name: Generate a Markdown table from the normalized YAML file
run: yml2md normalized.yml > table_data.md
- name: Generate a new software list from the updated data
run: md-from-template config/SOFTWARE-LIST.tpl.md table_data.md > SOFTWARE-LIST.md
- name: Update the YAML and Markdown files as appropriate
run: config/update_software_lists.sh
- id: commit-for-testing
uses: stefanzweifel/git-auto-commit-action@v4
with:
branch: ${{ needs.setup.outputs.testing_branch }}
commit_message: Update the software list
commit_message: Normalize YAML files and update the software lists
commit_user_name: ${{ needs.setup.outputs.git_user }}
commit_user_email: ${{ needs.setup.outputs.git_email }}
commit_author: ${{ needs.setup.outputs.git_author }}
file_pattern: SOFTWARE-LIST.md data/cisagov*.yml
merge_list_update:
file_pattern: data/cisagov*.yml software_lists/software_list_*.md
merge_updates:
runs-on: ubuntu-latest
needs:
- setup
- generate_list_update
if: needs.generate_list_update.outputs.updated_list == 'true'
- generate_updates
if: needs.generate_updates.outputs.has_updates == 'true'
steps:
- uses: actions/checkout@v2
with:

@ -46,3 +46,23 @@ repos:
- id: yamllint
args:
- --strict
# Shell script hooks
- repo: https://github.com/cisagov/pre-commit-shfmt
rev: v0.0.2
hooks:
- id: shfmt
args:
# Indent by two spaces
- -i
- '2'
# Binary operators may start a line
- -bn
# Switch cases are indented
- -ci
# Redirect operators are followed by a space
- -sr
- repo: https://github.com/detailyang/pre-commit-shell
rev: 1.0.5
hooks:
- id: shell-lint

@ -87,4 +87,4 @@ in [`PULL-EXAMPLE.md`](PULL-EXAMPLE.md).
## Software List ##
To view the full list of vendors & software [click here](./SOFTWARE-LIST.md).
To view the full list of vendors & software [click here](./software_lists/README.md).

File diff suppressed because it is too large Load Diff

@ -1,5 +1,15 @@
# CISA Log4j (CVE-2021-44228) Affected Vendor & Software List #
[0-9](software_list_Non-Alphabet.md) [A](software_list_A.md) [B](software_list_B.md)
[C](software_list_C.md) [D](software_list_D.md) [E](software_list_E.md)
[F](software_list_F.md) [G](software_list_G.md) [H](software_list_H.md)
[I](software_list_I.md) [J](software_list_J.md) [K](software_list_K.md)
[L](software_list_L.md) [M](software_list_M.md) [N](software_list_N.md)
[O](software_list_O.md) [P](software_list_P.md) [Q](software_list_Q.md)
[R](software_list_R.md) [S](software_list_S.md) [T](software_list_T.md)
[U](software_list_U.md) [V](software_list_V.md) [W](software_list_W.md)
[X](software_list_X.md) [Y](software_list_Y.md) [Z](software_list_Z.md)
## Status Descriptions ##
| Status | Description |

@ -0,0 +1,34 @@
#!/usr/bin/env bash
# This script is used to do the following:
# - Normalize each of the data/cisagov_*.yml files.
# - Update the comprehensive data/cisagov.yml file.
# - Generate each software_lists/software_list_*.md file from its respective
# data/cisagov_*.yml file.
set -o nounset
set -o errexit
set -o pipefail
OUTPUT_DIRECTORY=software_lists
TEMPLATE_FILE=config/SOFTWARE-LIST.tpl.md
echo Normalize individual cisagov YAML files
for file in data/cisagov_*.yml; do
echo " $file..."
normalize-yml --cisagov-format "$file" > "$file".tmp
mv --force "$file".tmp "$file"
done
echo Update the comprehensive cisagov YAML file
normalize-yml --cisagov-format data/cisagov_*.yml > data/cisagov.yml
echo Generate Markdown files from the individual cisagov YAML files
for file in data/cisagov_*.yml; do
echo " $file..."
# Convert the file path data/cisagov_*.yml to software_list_*.md
md_file=$(echo "$file" | sed 's/data\/cisagov_\(.\+\)yml/software_list_\1md/g')
normalize-yml "$file" > "$file.tmp"
yml2md "$file.tmp" > "$md_file.tmp"
md-from-template $TEMPLATE_FILE "$md_file.tmp" > "$OUTPUT_DIRECTORY/$md_file"
done

@ -0,0 +1,25 @@
# CISA Log4j (CVE-2021-44228) Affected Vendor & Software Lists #
## Contributing Guidelines ##
We welcome contributions! Please see [`CONTRIBUTING.md`](../CONTRIBUTING.md) for
details.
### Creating a pull request ###
Instructions for creating a pull request using the GitHub Web UI can be found
in [`PULL-EXAMPLE.md`](../PULL-EXAMPLE.md).
## Software Lists ##
To view the full lists of vendors & software click below:
[0-9](software_list_Non-Alphabet.md) [A](software_list_A.md) [B](software_list_B.md)
[C](software_list_C.md) [D](software_list_D.md) [E](software_list_E.md)
[F](software_list_F.md) [G](software_list_G.md) [H](software_list_H.md)
[I](software_list_I.md) [J](software_list_J.md) [K](software_list_K.md)
[L](software_list_L.md) [M](software_list_M.md) [N](software_list_N.md)
[O](software_list_O.md) [P](software_list_P.md) [Q](software_list_Q.md)
[R](software_list_R.md) [S](software_list_S.md) [T](software_list_T.md)
[U](software_list_U.md) [V](software_list_V.md) [W](software_list_W.md)
[X](software_list_X.md) [Y](software_list_Y.md) [Z](software_list_Z.md)
Loading…
Cancel
Save