2023-03-09 19:52:06 +00:00
|
|
|
#!/usr/bin/env python3
|
|
|
|
|
#
|
|
|
|
|
# SPDX-License-Identifier: GPL-2.0
|
|
|
|
|
#
|
|
|
|
|
# Copyright (c) 2013-2023 Igor Pecovnik, igor@armbian.com
|
|
|
|
|
#
|
|
|
|
|
# This file is a part of the Armbian Build Framework
|
|
|
|
|
# https://github.com/armbian/build/
|
|
|
|
|
#
|
armbian-next: Python patching delusion, pt1 & pt2 & pt3
- WiP: Python patching delusion, pt 1: finding & parsing patches; apply & git commit with pygit2; Markdown summaries (also for aggregation); git-to-patches tool
- Python: Markdown aggregation and patching summaries; collapsible; SummarizedMarkdownWriter
- Python: Markdown aggregation and patching summaries
- Python: reorg a bit into common/armbian_utils; define the `ASSET_LOG_BASE` in preparation for Markdown delusion
- Python patching: initial apply patches & initial commit patches to git (using pygit2)
- Python patching: add basic `series.conf` support
- Python patching: force use of utf-8; better error handling; use realpath of dirs
- Python patching: `git-to-patches` initial hack. not proud. half-reused some of the patches-to-git
- Python patching: "tag" the git commits with info for extracting later; introduce REWRITE_PATCHES/rewrite_patches_in_place
- Python patching: commented-out, recover-bad-patches hacks
- Python patching: shorten the signature
- Python patching: allow BASE_GIT_TAG as well as BASE_GIT_REVISION
- Python patching: git-archeology for patches missing descriptions; avoid UTF-8 in header/desc (not diff)
- Python patching: use modern-er email.utils.parsedate_to_datetime to parse commit date
- Python patching: unify PatchInPatchFile; better git-commiting; re-exporting patches from Git (directly)
- Python patching: switch to GitPython
- GitPython is like 100x slower than pygit2, but actually allows for date & committer
- also allows to remove untracked files before starting
- Python aggregation: fix missing `AGGREGATED_APT_SOURCES_DICT`
- Python patching: add `unidecode` dependency to pip3 install
- Python patching: don't try archeology if SRC is not a Git Repo (eg, in Docker)
- Python patching: don't try archeology if not applying patches to git
- WiP: Python patching delusion, pt2: actually use for u-boot & kernel patching
- Python patching: much better problem handling/logging; lenient with recreations (kernel)
- Python patching: don't force SHOW_LOG for u-boot patching
- Python patching: don't bomb for no reason when there are no patches to apply
- Python patching: fully (?) switch kernel patching to Python
- Python patching: more logging fixups
- Python patching: capture `kernel_git_revision` from `fetch_from_repo()`'s `checked_out_revision`
- Python patching: fully switch u-boot patching to Python
- Python aggregation/patching: colored logging; patching: always reset to git revision
- Python aggregation/patching: better logging; introduce u-boot Python patching
- Python patching pt3: recovers and better Markdown
- Python patching: detect, and rescue, `wrong_strip_level` problem; don't try to export patches that didn't apply, bitch instead
- Python patching: Markdown patching summary table, complete with emoji
- Python patching: include the problem breakdown in Markdown summary
- Python patching: sanity check against half-bare, half-mbox patches
- Python patching: try to recover from 1) bad utf-8 encoded patches; 2) bad unidiff patches; add a few sanity checks
- Python patching: try, and fail, to apply badly utf-8 encoded patches directly as bytes [reverted]
- Python patching: try to recover from patch *parse* failures; show summary; better logging
- set `GIT_ARCHEOLOGY=yes` to do archeology, default not
- armbian-next: Python `pip` dependencies handling, similar to `hostdeps`
- same scheme for Dockerfile caching
- @TODO: still using global/shared environment; should move to a dir under `cache` or some kinda venv
- WiP: add `python3-pip` to hostdeps; remove `python-setuptools`
- remove `python-setuptools` (Python2, no longer exists in Sid) from hostdeps
- add `python3-pip` to hostdeps; part of virtualenv saga
- WiP: split `kernel.sh` a bit, into `kernel-patching.sh`, `kernel-config.sh` and `kernel-make.sh`
- `advanced_patch()`: rename vars for clarity; no real changes
- Python patching: introduce FAST_ARCHEOLOGY; still trying for Markdown links
2022-12-01 15:09:26 +00:00
|
|
|
import logging
|
|
|
|
|
|
|
|
|
|
from . import armbian_utils as armbian_utils
|
|
|
|
|
|
|
|
|
|
log: logging.Logger = logging.getLogger("md_asset_log")
|
|
|
|
|
|
|
|
|
|
ASSET_LOG_BASE = armbian_utils.get_from_env("ASSET_LOG_BASE")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def write_md_asset_log(file: str, contents: str):
|
|
|
|
|
"""Log a message to the asset log file."""
|
|
|
|
|
if ASSET_LOG_BASE is None:
|
|
|
|
|
log.debug(f"ASSET_LOG_BASE not defined; here's the contents:\n{contents}")
|
|
|
|
|
return
|
|
|
|
|
target_file = ASSET_LOG_BASE + file
|
|
|
|
|
with open(target_file, "w") as asset_log:
|
|
|
|
|
asset_log.write(contents)
|
|
|
|
|
log.debug(f"- Wrote to {target_file}.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class SummarizedMarkdownWriter:
|
|
|
|
|
def __init__(self, file_name, title):
|
|
|
|
|
self.file_name = file_name
|
|
|
|
|
self.title = title
|
|
|
|
|
self.summary: list[str] = []
|
|
|
|
|
self.contents = ""
|
|
|
|
|
|
|
|
|
|
def __enter__(self):
|
|
|
|
|
return self
|
|
|
|
|
|
|
|
|
|
def __exit__(self, *args):
|
|
|
|
|
write_md_asset_log(self.file_name, self.get_summarized_markdown())
|
|
|
|
|
log.info(f"Summary: {self.title}: {'; '.join(self.summary)}")
|
|
|
|
|
|
|
|
|
|
def add_summary(self, summary):
|
|
|
|
|
self.summary.append(summary)
|
|
|
|
|
|
|
|
|
|
def write(self, text):
|
|
|
|
|
self.contents += text
|
|
|
|
|
|
2023-05-06 23:38:30 +00:00
|
|
|
def validate(self):
|
armbian-next: Python patching delusion, pt1 & pt2 & pt3
- WiP: Python patching delusion, pt 1: finding & parsing patches; apply & git commit with pygit2; Markdown summaries (also for aggregation); git-to-patches tool
- Python: Markdown aggregation and patching summaries; collapsible; SummarizedMarkdownWriter
- Python: Markdown aggregation and patching summaries
- Python: reorg a bit into common/armbian_utils; define the `ASSET_LOG_BASE` in preparation for Markdown delusion
- Python patching: initial apply patches & initial commit patches to git (using pygit2)
- Python patching: add basic `series.conf` support
- Python patching: force use of utf-8; better error handling; use realpath of dirs
- Python patching: `git-to-patches` initial hack. not proud. half-reused some of the patches-to-git
- Python patching: "tag" the git commits with info for extracting later; introduce REWRITE_PATCHES/rewrite_patches_in_place
- Python patching: commented-out, recover-bad-patches hacks
- Python patching: shorten the signature
- Python patching: allow BASE_GIT_TAG as well as BASE_GIT_REVISION
- Python patching: git-archeology for patches missing descriptions; avoid UTF-8 in header/desc (not diff)
- Python patching: use modern-er email.utils.parsedate_to_datetime to parse commit date
- Python patching: unify PatchInPatchFile; better git-commiting; re-exporting patches from Git (directly)
- Python patching: switch to GitPython
- GitPython is like 100x slower than pygit2, but actually allows for date & committer
- also allows to remove untracked files before starting
- Python aggregation: fix missing `AGGREGATED_APT_SOURCES_DICT`
- Python patching: add `unidecode` dependency to pip3 install
- Python patching: don't try archeology if SRC is not a Git Repo (eg, in Docker)
- Python patching: don't try archeology if not applying patches to git
- WiP: Python patching delusion, pt2: actually use for u-boot & kernel patching
- Python patching: much better problem handling/logging; lenient with recreations (kernel)
- Python patching: don't force SHOW_LOG for u-boot patching
- Python patching: don't bomb for no reason when there are no patches to apply
- Python patching: fully (?) switch kernel patching to Python
- Python patching: more logging fixups
- Python patching: capture `kernel_git_revision` from `fetch_from_repo()`'s `checked_out_revision`
- Python patching: fully switch u-boot patching to Python
- Python aggregation/patching: colored logging; patching: always reset to git revision
- Python aggregation/patching: better logging; introduce u-boot Python patching
- Python patching pt3: recovers and better Markdown
- Python patching: detect, and rescue, `wrong_strip_level` problem; don't try to export patches that didn't apply, bitch instead
- Python patching: Markdown patching summary table, complete with emoji
- Python patching: include the problem breakdown in Markdown summary
- Python patching: sanity check against half-bare, half-mbox patches
- Python patching: try to recover from 1) bad utf-8 encoded patches; 2) bad unidiff patches; add a few sanity checks
- Python patching: try, and fail, to apply badly utf-8 encoded patches directly as bytes [reverted]
- Python patching: try to recover from patch *parse* failures; show summary; better logging
- set `GIT_ARCHEOLOGY=yes` to do archeology, default not
- armbian-next: Python `pip` dependencies handling, similar to `hostdeps`
- same scheme for Dockerfile caching
- @TODO: still using global/shared environment; should move to a dir under `cache` or some kinda venv
- WiP: add `python3-pip` to hostdeps; remove `python-setuptools`
- remove `python-setuptools` (Python2, no longer exists in Sid) from hostdeps
- add `python3-pip` to hostdeps; part of virtualenv saga
- WiP: split `kernel.sh` a bit, into `kernel-patching.sh`, `kernel-config.sh` and `kernel-make.sh`
- `advanced_patch()`: rename vars for clarity; no real changes
- Python patching: introduce FAST_ARCHEOLOGY; still trying for Markdown links
2022-12-01 15:09:26 +00:00
|
|
|
if len(self.title) == 0:
|
|
|
|
|
raise Exception("Markdown Summary Title not set")
|
|
|
|
|
if len(self.summary) == 0:
|
|
|
|
|
raise Exception("Markdown Summary not set")
|
|
|
|
|
if self.contents == "":
|
|
|
|
|
raise Exception("Markdown Contents not set")
|
2023-05-06 23:38:30 +00:00
|
|
|
|
|
|
|
|
# see https://docs.github.com/en/get-started/writing-on-github/working-with-advanced-formatting/organizing-information-with-collapsed-sections
|
|
|
|
|
def get_summarized_markdown(self):
|
|
|
|
|
self.validate()
|
armbian-next: Python patching delusion, pt1 & pt2 & pt3
- WiP: Python patching delusion, pt 1: finding & parsing patches; apply & git commit with pygit2; Markdown summaries (also for aggregation); git-to-patches tool
- Python: Markdown aggregation and patching summaries; collapsible; SummarizedMarkdownWriter
- Python: Markdown aggregation and patching summaries
- Python: reorg a bit into common/armbian_utils; define the `ASSET_LOG_BASE` in preparation for Markdown delusion
- Python patching: initial apply patches & initial commit patches to git (using pygit2)
- Python patching: add basic `series.conf` support
- Python patching: force use of utf-8; better error handling; use realpath of dirs
- Python patching: `git-to-patches` initial hack. not proud. half-reused some of the patches-to-git
- Python patching: "tag" the git commits with info for extracting later; introduce REWRITE_PATCHES/rewrite_patches_in_place
- Python patching: commented-out, recover-bad-patches hacks
- Python patching: shorten the signature
- Python patching: allow BASE_GIT_TAG as well as BASE_GIT_REVISION
- Python patching: git-archeology for patches missing descriptions; avoid UTF-8 in header/desc (not diff)
- Python patching: use modern-er email.utils.parsedate_to_datetime to parse commit date
- Python patching: unify PatchInPatchFile; better git-commiting; re-exporting patches from Git (directly)
- Python patching: switch to GitPython
- GitPython is like 100x slower than pygit2, but actually allows for date & committer
- also allows to remove untracked files before starting
- Python aggregation: fix missing `AGGREGATED_APT_SOURCES_DICT`
- Python patching: add `unidecode` dependency to pip3 install
- Python patching: don't try archeology if SRC is not a Git Repo (eg, in Docker)
- Python patching: don't try archeology if not applying patches to git
- WiP: Python patching delusion, pt2: actually use for u-boot & kernel patching
- Python patching: much better problem handling/logging; lenient with recreations (kernel)
- Python patching: don't force SHOW_LOG for u-boot patching
- Python patching: don't bomb for no reason when there are no patches to apply
- Python patching: fully (?) switch kernel patching to Python
- Python patching: more logging fixups
- Python patching: capture `kernel_git_revision` from `fetch_from_repo()`'s `checked_out_revision`
- Python patching: fully switch u-boot patching to Python
- Python aggregation/patching: colored logging; patching: always reset to git revision
- Python aggregation/patching: better logging; introduce u-boot Python patching
- Python patching pt3: recovers and better Markdown
- Python patching: detect, and rescue, `wrong_strip_level` problem; don't try to export patches that didn't apply, bitch instead
- Python patching: Markdown patching summary table, complete with emoji
- Python patching: include the problem breakdown in Markdown summary
- Python patching: sanity check against half-bare, half-mbox patches
- Python patching: try to recover from 1) bad utf-8 encoded patches; 2) bad unidiff patches; add a few sanity checks
- Python patching: try, and fail, to apply badly utf-8 encoded patches directly as bytes [reverted]
- Python patching: try to recover from patch *parse* failures; show summary; better logging
- set `GIT_ARCHEOLOGY=yes` to do archeology, default not
- armbian-next: Python `pip` dependencies handling, similar to `hostdeps`
- same scheme for Dockerfile caching
- @TODO: still using global/shared environment; should move to a dir under `cache` or some kinda venv
- WiP: add `python3-pip` to hostdeps; remove `python-setuptools`
- remove `python-setuptools` (Python2, no longer exists in Sid) from hostdeps
- add `python3-pip` to hostdeps; part of virtualenv saga
- WiP: split `kernel.sh` a bit, into `kernel-patching.sh`, `kernel-config.sh` and `kernel-make.sh`
- `advanced_patch()`: rename vars for clarity; no real changes
- Python patching: introduce FAST_ARCHEOLOGY; still trying for Markdown links
2022-12-01 15:09:26 +00:00
|
|
|
return f"<details><summary>{self.title}: {'; '.join(self.summary)}</summary>\n<p>\n\n{self.contents}\n\n</p></details>\n"
|
2022-12-20 15:17:21 +00:00
|
|
|
|
|
|
|
|
def get_readme_markdown(self):
|
2023-05-06 23:38:30 +00:00
|
|
|
self.validate()
|
2022-12-20 15:17:21 +00:00
|
|
|
return f"#### {self.title}: {'; '.join(self.summary)}\n\n{self.contents}\n\n"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def get_gh_pages_workflow_script():
|
|
|
|
|
return """
|
|
|
|
|
name: publish-ghpages
|
|
|
|
|
|
2023-01-25 19:51:57 +00:00
|
|
|
on:
|
|
|
|
|
workflow_dispatch:
|
|
|
|
|
push:
|
|
|
|
|
branches:
|
|
|
|
|
- '*'
|
2022-12-20 15:17:21 +00:00
|
|
|
|
|
|
|
|
jobs:
|
|
|
|
|
publish:
|
|
|
|
|
runs-on: ubuntu-latest
|
|
|
|
|
steps:
|
|
|
|
|
# Do NOT checkout this. It is a kernel tree and takes a long time, and it's not necessary.
|
|
|
|
|
- name: Grab README.md
|
|
|
|
|
env:
|
|
|
|
|
BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
|
|
|
|
|
run: |
|
|
|
|
|
curl -s https://raw.githubusercontent.com/${{ github.repository }}/${BRANCH_NAME}/README.md > README.md
|
|
|
|
|
ls -la README.md
|
2023-05-06 23:38:30 +00:00
|
|
|
|
2022-12-20 15:17:21 +00:00
|
|
|
# install grip via pip, https://github.com/joeyespo/grip; rpardini's fork https://github.com/rpardini/grip
|
|
|
|
|
- name: Install grip
|
|
|
|
|
run: |
|
|
|
|
|
pip3 install https://github.com/rpardini/grip/archive/refs/heads/master.tar.gz
|
|
|
|
|
|
|
|
|
|
- name: Run grip to gen ${{ github.head_ref || github.ref_name }}
|
|
|
|
|
env:
|
|
|
|
|
BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
|
|
|
|
|
run: |
|
|
|
|
|
mkdir -p public
|
|
|
|
|
grip README.md --context=${{ github.repository }} --title="${BRANCH_NAME}" --wide --user-content --export "public/${BRANCH_NAME}.html" || true
|
|
|
|
|
ls -la public/
|
|
|
|
|
|
|
|
|
|
- name: Deploy to GitHub Pages (gh-pages branch)
|
|
|
|
|
if: success()
|
|
|
|
|
uses: crazy-max/ghaction-github-pages@v3
|
|
|
|
|
with:
|
|
|
|
|
target_branch: gh-pages
|
|
|
|
|
build_dir: public
|
|
|
|
|
keep_history: true
|
|
|
|
|
jekyll: false
|
|
|
|
|
env:
|
|
|
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
|
|
|
"""
|