Rewrite ffmpeg-free tarball generation scripts

They are split in two scripts now. One you can use to update the file list from a build. However this will only cover files for your architecture.
3 years ago · ba66d3f0e5
parent 40c8764fc5
commit ba66d3f0e5
7 changed files with 3265 additions and 1246 deletions
--- a/README.md
+++ b/README.md
@ -8,8 +8,11 @@ community or a corporation.

 ## Creating the 'free' tarball

-1. Update the `Version` in the spec file
-2. Set the `Release` to 0
-2. Do a full build locally: `fedpkg mockbuild --with full_build`
-3. Create the 'clean' tarball: `./ffmpeg_clean_sources.sh results_ffmpeg/5.0/0.fc35/build.log`
-4. Set the `Release` to 1
+1. Update the `Version` in the spec file.
+2. Set the `Release` to 0 in the spec file.
+3. Do a full build locally: `fedpkg mockbuild --with full_build`
+4. Run `./ffmpeg_update_free_sources.sh results_ffmpeg/5.0/0.fc35/build.log`
+   This will update the `ffmpeg_free_sources` file list.
+5. Run `./ffmpeg_gen_free_tarball.sh` to create the tarball.
+6. Set the `Release` to 1 in the spec file.
+7. Do a scratch build.
--- a/ffmpeg.spec
+++ b/ffmpeg.spec
@ -74,7 +74,7 @@ Source1:        https://ffmpeg.org/releases/ffmpeg-%{version}.tar.xz.asc
 # https://ffmpeg.org/ffmpeg-devel.asc
 # gpg2 --import --import-options import-export,import-minimal ffmpeg-devel.asc > ./ffmpeg.keyring
 Source2:        ffmpeg.keyring
-Source3:        ffmpeg_clean_sources.sh
+Source3:        ffmpeg_free_sources
 Source20:       enable_decoders
 Source21:       enable_encoders
 # Change path from /usr/local to /usr
--- a/ffmpeg_clean_sources.sh
+++ b/ffmpeg_clean_sources.sh
--- a/3038
+++ b/3038
--- a/ffmpeg_gen_free_tarball.sh
+++ b/ffmpeg_gen_free_tarball.sh
@ -0,0 +1,128 @@
+#!/bin/bash
+#
+# Copyright (c) 2022      Andreas Schneider <asn@cryptomilk.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# shellcheck disable=2181
+
+FF_PKGNAME="ffmpeg"
+FF_PKGNAME_SUFFIX="-free"
+FF_VERSION="$(rpmspec -P ./*.spec | grep ^Version | sed -e 's/Version:[ ]*//g')"
+FF_TARBALL_URL="https://ffmpeg.org/releases/${FF_PKGNAME}-${FF_VERSION}.tar.xz"
+FF_TARBALL="$(basename "${FF_TARBALL_URL}")"
+FF_GPG_ARMOR_FILE="${FF_TARBALL}.asc"
+FF_PKG_DIR="$(pwd)"
+FF_KEYRING="${FF_PKG_DIR}/ffmpeg.keyring"
+FF_TMPDIR=$(mktemp --tmpdir -d ffmpeg-XXXXXXXX)
+FF_PATH="${FF_TMPDIR}/${FF_PKGNAME}-${FF_VERSION}"
+
+cleanup_tmpdir() {
+    # shellcheck disable=2164
+    popd 2>/dev/null
+    rm -rf "${FF_TMPDIR}"
+}
+trap cleanup_tmpdir SIGINT
+
+cleanup_and_exit()
+{
+    cleanup_tmpdir
+
+    if test "$1" = 0 -o -z "$1"; then
+        exit 0
+    else
+        # shellcheck disable=2086
+        exit ${1}
+    fi
+}
+
+if [[ ! -w "${FF_TARBALL}" ]]; then
+    echo ">>> Downloading tarball"
+    wget "${FF_TARBALL_URL}"
+fi
+if [[ ! -w "${FF_TARBALL}.asc" ]]; then
+    echo ">>> Downloading signature"
+    wget "${FF_TARBALL_URL}.asc"
+fi
+
+echo ">>> Verifying ${FF_TARBALL} GPG signature"
+gpgv2 --quiet --keyring "${FF_KEYRING}" "${FF_GPG_ARMOR_FILE}" "${FF_TARBALL}"
+if [ $? -ne 0 ]; then
+    echo "ERROR: GPG signature verification failed"
+    cleanup_and_exit 1
+fi
+echo
+
+echo ">>> Unpacking ${FF_TARBALL}"
+
+tar -xf "${FF_TARBALL}" -C "${FF_TMPDIR}"
+if [ $? -ne 0 ]; then
+    echo "ERROR: Failed to unpack ${FF_TARBALL}"
+    cleanup_and_exit 1
+fi
+
+if [[ ! -r ffmpeg_free_sources ]]; then
+    echo "ERROR: ffmpeg_free_sources doesn't exist!"
+    cleanup_and_exit 1
+fi
+readarray -t keepfiles < ffmpeg_free_sources
+
+pushd "${FF_PATH}" || cleanup_and_exit 1
+
+echo
+echo ">>> Cleaning up sources for new tarball ..."
+
+# Get file list from ffmpeg
+mapfile -d '' filelist < <(find ./ -type f -printf '%P\0')
+
+# Sort arrays
+readarray -t keepfiles_sorted < <(printf '%s\0' "${keepfiles[@]}" | sort -z | xargs -0n1)
+readarray -t filelist_sorted < <(printf '%s\0' "${filelist[@]}" | sort -z | xargs -0n1)
+
+# Compare arrays and remove files which are left over
+comm -2 -3 -z <(printf '%s\0' "${filelist_sorted[@]}") <(printf '%s\0' "${keepfiles_sorted[@]}") | xargs -0 rm -f
+
+readarray -t removed_files < <(comm -1 -3 -z <(printf '%s\0' "${filelist_sorted[@]}") <(printf '%s\0' "${keepfiles_sorted[@]}") | xargs -0n1)
+if [[ "${#removed_files[@]}" -ge 1 ]]; then
+    if [[ "${#removed_files[@]}" -eq 1 ]] && [[ -z "${removed_files[0]}" ]]; then
+        echo "... done"
+    else
+        echo "File not in upstream tarball anymore (please cleanup 'ffmpeg_free_sources'):"
+        for f in "${removed_files[@]}"; do
+            if [[ -z "${f}" ]]; then
+                continue
+            fi
+            echo "  * ${f}"
+        done
+    fi
+fi
+echo
+
+popd || cleanup_and_exit 1 # /FF_PATH
+
+pushd "${FF_TMPDIR}" || cleanup_and_exit 1
+
+echo ">>> Create new tarball ${FF_PKGNAME}${FF_PKGNAME_SUFFIX}-${FF_VERSION}.tar.xz ..."
+tar -cJf "${FF_PKG_DIR}/${FF_PKGNAME}${FF_PKGNAME_SUFFIX}-${FF_VERSION}.tar.xz" "${FF_PKGNAME}-${FF_VERSION}"
+if [ $? -ne 0 ]; then
+    echo "ERROR: Creating tarball failed"
+    cleanup_and_exit 1
+fi
+
+popd || cleanup_and_exit 1 # /FF_TMPDIR
+
+du -sh "${FF_PKGNAME}${FF_PKGNAME_SUFFIX}-${FF_VERSION}.tar.xz"
+echo
+
+cleanup_and_exit 0
--- a/ffmpeg_update_free_sources.sh
+++ b/ffmpeg_update_free_sources.sh
@ -0,0 +1,89 @@
+#!/bin/bash
+#
+# Copyright (c) 2022      Andreas Schneider <asn@cryptomilk.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# shellcheck disable=2181
+
+FF_PKGNAME="ffmpeg"
+FF_PKGNAME_SUFFIX="-free"
+FF_VERSION="$(rpmspec -P ./*.spec | grep ^Version | sed -e 's/Version:[ ]*//g')"
+FF_TARBALL_URL="https://ffmpeg.org/releases/${FF_PKGNAME}-${FF_VERSION}.tar.xz"
+FF_TARBALL="$(basename "${FF_TARBALL_URL}")"
+FF_GPG_ARMOR_FILE="${FF_TARBALL}.asc"
+FF_PKG_DIR="$(pwd)"
+FF_KEYRING="${FF_PKG_DIR}/ffmpeg.keyring"
+FF_TMPDIR=$(mktemp --tmpdir -d ffmpeg-XXXXXXXX)
+FF_PATH="${FF_TMPDIR}/${FF_PKGNAME}-${FF_VERSION}"
+
+cleanup_tmpdir() {
+    # shellcheck disable=2164
+    popd 2>/dev/null
+    rm -rf "${FF_TMPDIR}"
+}
+trap cleanup_tmpdir SIGINT
+
+cleanup_and_exit()
+{
+    cleanup_tmpdir
+
+    if test "$1" = 0 -o -z "$1"; then
+        exit 0
+    else
+        # shellcheck disable=2086
+        exit ${1}
+    fi
+}
+
+function usage()
+{
+    echo "Usage: $(basename "${0}") BUILD_LOG"
+    cleanup_and_exit 0
+}
+
+if [[ $# -lt 1 ]]; then
+    usage
+    cleanup_and_exit 0
+fi
+
+echo ">>> Collect information from ${1}"
+build_log="$(readlink -f "${1}")"
+if [[ -z "${build_log}" ]] || [[ ! -r "${build_log}" ]]; then
+    echo "Build log doesn't exist: %{build_log}"
+    cleanup_and_exit 1
+fi
+
+asm_files="$(grep "^gcc.*\.c$" "${build_log}" | awk 'NF>1{print $NF}' | sort)"
+c_files="$(grep "^nasm.*\.asm$" "${build_log}" | awk 'NF>1{print $NF}' | sort)"
+
+# shellcheck disable=2206
+new_sources=(${asm_files}
+             ${c_files})
+
+# Sort arrays
+readarray -t new_sources_sorted < <(printf '%s\0' "${new_sources[@]}" | sort -z | xargs -0n1)
+
+# Create a backup for a diff
+cp -a ffmpeg_free_sources ffmpeg_free_sources.orig
+cp -a ffmpeg_free_sources ffmpeg_free_sources.new
+printf "%s\n" "${new_sources_sorted[@]}" >> ffmpeg_free_sources.new
+# Update ffmpeg_free_sources
+echo ">>> Updating ffmpeg_free_sources"
+sort < ffmpeg_free_sources.new | uniq | sed '/^$/d' > ffmpeg_free_sources
+echo ">>> Differences in file list"
+diff -u ffmpeg_free_sources.orig ffmpeg_free_sources
+rm -f ffmpeg_free_sources.new
+
+cleanup_and_exit 0
--- a/2
+++ b/2
@ -1,2 +1,2 @@
-SHA512 (ffmpeg-free-5.0.tar.xz) = 547d4dc9c90e00a6896e7ffd6c67f98e66c9d451655268570a72680655172ec97d85e761bc743906b85982b8768a155c9d2ddd67991633557bbaa32052902cfd
+SHA512 (ffmpeg-free-5.0.tar.xz) = a7d2b386c46b8c66983669da6dec19cc051a9fb45ec0cd2975723e26a1f7a430ead5c2491ca09c05dff9d6097da98bb0d28956a39ac983e423beddc7e83c11d5
 SHA512 (ffmpeg-5.0.tar.xz.asc) = 9238fcdc849895147a26669ac90fec94002af86e81270128f6eb0448d74487d8ba06ee08d10686a7a6e204689716ee744bfffc2a39466940940609cfcd7dfa85