Commit some external source archives (for build stability)

.dockerignore

@@ -1,7 +1,6 @@
 Dockerfile
 build/
 build-*/
-sources
 minetest
 www
 .git

.github/workflows/build.yml

@@ -11,4 +11,4 @@ jobs:
       - name: Build docker image
         run: |
           docker build . -t mtweb
-          docker run --rm mtweb node --experimental-wasm-threads /minetest-wasm/www/minetest.js
+          docker run --rm mtweb ls -la /minetest-wasm/www

.gitignore

@@ -1,7 +1,12 @@
 minetest/
 build/
 build-*/
-sources/
 install/
 www/
 upload.sh
+sources/zlib
+sources/libjpeg
+sources/libpng
+sources/freetype
+sources/zstd
+sources/webshims

build_all.sh

@@ -9,6 +9,8 @@ rm -rf "$WWW_DIR"
 # with pthread support. Wipe the cache and build them ourselves.
 emcc --clear-cache --clear-ports
 
+./fetch_sources.sh
+
 # Dependencies
 ./build_zlib.sh
 ./build_libjpeg.sh

build_curl.sh

@@ -2,31 +2,16 @@
 
 source common.sh
 
-UPSTREAM="https://curl.se/download/curl-7.82.0.tar.bz2"
-TARBALL="curl-7.82.0.tar.bz2"
-TARDIR="curl-7.82.0"
-
-if [ ! -f sources/"$TARBALL" ]; then
-  pushd sources
-  wget "$UPSTREAM" -O "$TARBALL"
-  popd
-fi
-
-if ! sha256sum sources/"$TARBALL" | grep -q 46d9a0400a33408fd992770b04a44a7434b3036f2e8089ac28b57573d59d371f; then
-  echo "Wrong checksum for $TARDIR"
-  exit 1
-fi
-
-pushd "$BUILD_DIR"
-rm -rf "$TARDIR"
-tar -jxvf "$SRC_DIR/$TARBALL"
+unpack_source curl
 
 # Wrap socket ops
-$SRC_DIR/webshims/src/emsocket/wrap.py "$TARDIR"
+"$SOURCES_DIR/webshims/src/emsocket/wrap.py" "$BUILD_DIR/curl"
 
-rm -rf curl
-mkdir curl
-pushd curl
+pushd "$BUILD_DIR"
+
+rm -rf curl-build
+mkdir curl-build
+pushd curl-build
 
 # For emsocket.h
 export CFLAGS="-I${INSTALL_DIR}/include"
@@ -43,7 +28,7 @@ emcmake cmake \
   -DBUILD_CURL_EXE=OFF \
   -DBUILD_SHARED_LIBS=OFF \
   -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" \
-  "$BUILD_DIR/$TARDIR"
+  "$BUILD_DIR/curl"
 
 emmake make -j4
 emmake make install

build_freetype.sh

@@ -2,17 +2,11 @@
 
 source common.sh
 
-if [ ! -d sources/freetype ]; then
-  pushd sources
-  git clone https://gitlab.freedesktop.org/freetype/freetype.git freetype
-  popd
-fi
-
 pushd "$BUILD_DIR"
-rm -rf freetype
-mkdir freetype
+rm -rf freetype-build
+mkdir freetype-build
 
-pushd freetype
+pushd freetype-build
 
 emcmake cmake \
   -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" \
@@ -20,7 +14,7 @@ emcmake cmake \
   -DZLIB_INCLUDE_DIR="$INSTALL_DIR/include" \
   -DPNG_LIBRARY="$INSTALL_DIR/lib/libpng.a" \
   -DPNG_PNG_INCLUDE_DIR="$INSTALL_DIR/include" \
-  "$SRC_DIR/freetype"
+  "$SOURCES_DIR/freetype"
 emmake make
 emmake make install
 

build_fsroot.sh

@@ -2,33 +2,17 @@
 
 # Build virtual file system
 #
-# All the files minetest needs to function correctly.
+# The files minetest needs to function correctly.
 #
 # Shaders, fonts, games, etc
-#
-# The only source for this seems to be the official distribution.
-# There must be a script to build it somewhere, but I haven't found it.
 
 source common.sh
 
-UPSTREAM="https://github.com/minetest/minetest/releases/download/5.5.0/minetest-5.5.0-win64.zip"
-ZIPFILE="minetest-5.5.0-win64.zip"
-ZIPDIR="minetest-5.5.0-win64"
-
-if [ ! -d "$MINETEST_REPO" ] || [ ! -d "$IRRLICHT_REPO" ; then
-    echo "Minetest source not found"
-    exit 1
-fi
-
-if [ ! -f sources/"$ZIPFILE" ]; then
-    pushd sources
-    wget "$UPSTREAM"
-    popd
-fi
+# TODO: Use `make package` for this instead.
+MTDIR="minetest-5.5.1-win64"
+unpack_source $MTDIR
 
 pushd "$BUILD_DIR"
-rm -rf "$ZIPDIR"
-unzip "$SRC_DIR"/"$ZIPFILE"
 
 rm -rf fsroot
 mkdir fsroot
@@ -38,7 +22,7 @@ mkdir -p fsroot/etc/ssl/certs
 # ca-certificates.crt may be a symlink
 cat /etc/ssl/certs/ca-certificates.crt > fsroot/etc/ssl/certs/ca-certificates.crt
 
-mv "$ZIPDIR" fsroot/minetest
+mv $MTDIR fsroot/minetest
 
 pushd fsroot/minetest
 

build_libjpeg.sh

@@ -2,12 +2,6 @@
 
 source common.sh
 
-if [ ! -d sources/libjpeg ]; then
-  pushd sources
-  git clone "https://github.com/libjpeg-turbo/libjpeg-turbo.git" libjpeg
-  popd
-fi
-
 pushd "$BUILD_DIR"
 rm -rf libjpeg
 mkdir -p libjpeg
@@ -16,7 +10,7 @@ pushd libjpeg
 emcmake cmake \
 -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" \
 -DWITH_SIMD=0 \
-"$SRC_DIR/libjpeg"
+"$SOURCES_DIR/libjpeg"
 
 emmake make
 emmake make install

build_libogg.sh

@@ -2,26 +2,9 @@
 
 source common.sh
 
-TARNAME="libogg-1.3.5"
-TARBALL="libogg-1.3.5.tar.gz"
+unpack_source libogg
 
-if [ ! -f sources/"$TARBALL" ]; then
-  pushd sources
-  wget "https://downloads.xiph.org/releases/ogg/$TARBALL"
-  popd
-fi
-
-if ! sha256sum sources/"$TARBALL" | grep -q 0eb4b4b9420a0f51db142ba3f9c64b333f826532dc0f48c6410ae51f4799b664; then
-  echo "Wrong checksum for $TARNAME"
-  exit 1
-fi
-
-pushd "$BUILD_DIR"
-
-rm -rf "$TARNAME"
-tar -zxvf "$SRC_DIR/$TARBALL"
-
-pushd "$TARNAME"
+pushd "$BUILD_DIR/libogg"
 
 emconfigure ./configure --disable-shared --prefix="$INSTALL_DIR"
 emmake make

build_libpng.sh

@@ -2,15 +2,9 @@
 
 source common.sh
 
-if [ ! -d sources/libpng ]; then
-  pushd sources
-  git clone https://git.code.sf.net/p/libpng/code libpng
-  popd
-fi
-
 pushd "$BUILD_DIR"
 rm -rf libpng
-cp -a "$SRC_DIR/libpng" libpng
+cp -a "$SOURCES_DIR/libpng" libpng
 
 pushd libpng
 # For zlib

build_libvorbis.sh

@@ -2,26 +2,9 @@
 
 source common.sh
 
-TARNAME="libvorbis-1.3.7"
-TARBALL="libvorbis-1.3.7.tar.gz"
-
-if [ ! -f "sources/$TARBALL" ]; then
-  pushd sources
-  wget "https://downloads.xiph.org/releases/vorbis/$TARBALL"
-  popd
-fi
-
-if ! sha256sum "sources/$TARBALL" | grep -q 0e982409a9c3fc82ee06e08205b1355e5c6aa4c36bca58146ef399621b0ce5ab; then
-  echo "Wrong checksum"
-  exit 1
-fi
-
-pushd "$BUILD_DIR"
-rm -rf "$TARNAME"
-tar -zxvf "$SRC_DIR/$TARBALL"
-
-pushd "$TARNAME"
+unpack_source libvorbis
 
+pushd "$BUILD_DIR/libvorbis"
 emconfigure ./configure --disable-shared --prefix="$INSTALL_DIR" --with-ogg="$INSTALL_DIR"
 emmake make
 emmake make install

build_openssl.sh

@@ -2,29 +2,12 @@
 
 source common.sh
 
-URL="https://www.openssl.org/source/openssl-1.1.1n.tar.gz"
-TARBALL="openssl-1.1.1n.tar.gz"
-TARDIR="openssl-1.1.1n"
+unpack_source openssl
 
-if [ ! -f sources/"$TARBALL" ]; then
-  pushd sources
-  wget "$URL" -O "$TARBALL"
-  popd
-fi
+pushd "$BUILD_DIR/openssl"
 
-if ! sha256sum sources/"$TARBALL" | grep -q 40dceb51a4f6a5275bde0e6bf20ef4b91bfc32ed57c0552e2e8e15463372b17a; then
-  echo "Wrong checksum for $URL"
-  exit 1
-fi
+"$SOURCES_DIR/webshims/src/emsocket/wrap.py" .
 
-pushd "$BUILD_DIR"
-
-rm -rf "$TARDIR"
-tar -zxvf "$SRC_DIR/$TARBALL"
-
-$SRC_DIR/webshims/src/emsocket/wrap.py "$TARDIR"
-
-pushd "$TARDIR"
 patch -p1 < "$BASE_DIR"/openssl.patch
 
 export CFLAGS="-I${INSTALL_DIR}/include -DPEDANTIC"

build_sqlite3.sh

@@ -2,17 +2,9 @@
 
 source common.sh
 
-if [ ! -f sources/sqlite.tar.gz ]; then
-  pushd sources
-  wget "https://www.sqlite.org/src/tarball/sqlite.tar.gz?r=release" -O sqlite.tar.gz
-  popd
-fi
+unpack_source SQLite
 
-pushd "$BUILD_DIR"
-rm -rf sqlite
-tar -zxvf "$SRC_DIR/sqlite.tar.gz"
-
-pushd sqlite
+pushd "$BUILD_DIR/SQLite"
 export BUILD_CC="gcc"
 emconfigure ./configure --disable-shared --prefix="$INSTALL_DIR" cross_compiling=yes
 emmake make

build_webshims.sh

@@ -1,37 +1,13 @@
 #!/bin/bash -eux
 
-WEBSHIMS_REV=01b29b520fa2c97f9c8f79b4bfd5ae67ecba6e06
-
 source common.sh
 
-if [ ! -d sources/webshims ]; then
-  pushd sources
-  git clone "https://github.com/paradust7/webshims.git" webshims
-  popd
-fi
-
-pushd sources/webshims
-REV=`git rev-parse HEAD`
-popd
-
-if [ "$REV" != "$WEBSHIMS_REV" ]; then
-    set +x
-    echo "---------------------------------------------------------------"
-    echo "ERROR: sources/webshim on wrong revision"
-    echo "--------------------------------------------------------------"
-    echo "Expected revision: $WEBSHIMS_REV"
-    echo "Actual revision: $REV"
-    echo "---------------------------------------------------------------"
-    echo "Please 'git pull' or delete this directory before proceeding"
-    exit 1
-fi
-
 pushd "$BUILD_DIR"
-rm -rf webshims
-mkdir webshims
-pushd webshims
+rm -rf webshims-build
+mkdir webshims-build
+pushd webshims-build
 
-emcmake cmake -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" "$SRC_DIR/webshims"
+emcmake cmake -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" "$SOURCES_DIR/webshims"
 emmake make
 emmake make install
 

build_zlib.sh

@@ -2,15 +2,9 @@
 
 source common.sh
 
-if [ ! -d sources/zlib ]; then
-  pushd sources
-  git clone "https://github.com/madler/zlib.git" zlib
-  popd
-fi
-
 pushd "$BUILD_DIR"
 rm -rf zlib
-cp -a "$SRC_DIR/zlib" zlib
+cp -a "$SOURCES_DIR/zlib" zlib
 
 pushd zlib
 emconfigure ./configure --static --prefix="$INSTALL_DIR"

build_zstd.sh

@@ -2,27 +2,16 @@
 
 source common.sh
 
-if [ ! -d sources/zstd ]; then
-  pushd sources
-  git clone https://github.com/facebook/zstd.git zstd
-
-  pushd zstd
-  git checkout v1.5.2
-  popd
-
-  popd
-fi
-
 pushd "$BUILD_DIR"
-rm -rf zstd
-mkdir zstd
+rm -rf zstd-build
+mkdir zstd-build
 
-pushd zstd
+pushd zstd-build
 export CFLAGS="-D_POSIX_SOURCE=1"
 export CXXFLAGS="-D_POSIX_SOURCE=1"
 emcmake cmake \
   -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" \
-  "$SRC_DIR/zstd/build/cmake"
+  "$SOURCES_DIR/zstd/build/cmake"
 
 emmake make
 emmake make install

common.sh

@@ -28,11 +28,11 @@ case $BUILD_KIND in
     exit 1
 esac
 
-SRC_DIR="$BASE_DIR/sources"
+SOURCES_DIR="$BASE_DIR/sources"
 BUILD_DIR="$BASE_DIR/build$BUILD_SUFFIX"
 INSTALL_DIR="$BUILD_DIR/install"
 WWW_DIR="$BASE_DIR/www"
-mkdir -p "$SRC_DIR" "$BUILD_DIR" "$INSTALL_DIR"
+mkdir -p "$SOURCES_DIR" "$BUILD_DIR" "$INSTALL_DIR"
 
 export CFLAGS="$COMMON_CFLAGS -pthread -sUSE_PTHREADS=1 -fexceptions"
 export CXXFLAGS="$COMMON_CFLAGS -pthread -sUSE_PTHREADS=1 -fexceptions"
@@ -44,3 +44,115 @@ export EMSDK_SYSINCLUDE="${EMSDK_ROOT}/upstream/emscripten/cache/sysroot/include
 
 export MINETEST_REPO="$BASE_DIR/minetest"
 export IRRLICHT_REPO="$BASE_DIR/minetest/lib/irrlichtmt"
+
+function getsource() {
+  local url="$1"
+  local hsh="$2"
+  local filename="$(basename -- "${url%%\?*}")"
+
+  pushd "$SOURCES_DIR"
+  if [ ! -f "$filename" ]; then
+    wget "$url" -O "$filename"
+  fi
+  if ! sha256sum "$filename" | grep -q "$hsh"; then
+    echo "Wrong sha256 checksum for $filename"
+    exit 1
+  fi
+  popd
+}
+
+function do_unpack() {
+  local filename="$1"
+  case "$filename" in
+    *.tar.gz|*.tar.bz2|*.tar.xz|*.tgz)
+        tar xvf "$filename"
+        ;;
+    *.zip)
+        unzip "$filename"
+        ;;
+    *)
+        echo "Not sure how to unpack: $filename"
+        exit 1
+        ;;
+  esac
+}
+
+function strip_ext() {
+  local filename="$1"
+  case "$filename" in
+    *.tar.gz)  echo "${filename%.tar.gz}" ;;
+    *.tar.xz)  echo "${filename%.tar.xz}" ;;
+    *.tar.bz2) echo "${filename%.tar.bz2}" ;;
+    *.zip)     echo "${filename%.zip}" ;;
+    *)         echo "$filename" ;;
+  esac
+}
+
+# Usage:
+#
+#   unpack_source $prefix
+#
+# Finds the tar/zip file in sources/ with a specific prefix,
+# and untars/unzips it to the build directory, renamed as $prefix.
+function unpack_source() {
+  local prefix="$1"
+  pushd "$SOURCES_DIR"
+  shopt -s nullglob
+  local matches=("$prefix"*)
+  shopt -u nullglob
+  local count="${#matches[@]}"
+  popd
+
+  if [ $count -eq 0 ] ; then
+    echo "unpack_source $prefix: Could not find source tar/zip file"
+    exit 1
+  elif [ $count -ne 1 ]; then
+    echo "unpack_source $prefix: Ambiguous prefix (count=$count)"
+    exit 1
+  fi
+  local filename="${matches[0]}"
+  local dirname="$(strip_ext "$filename")"
+
+  pushd "$BUILD_DIR"
+  rm -rf "$prefix" "$dirname"
+  do_unpack "$SOURCES_DIR/$filename"
+  if [ ! -d "$dirname" ]; then
+    echo "Unpacking $filename did not produce the expected directory"
+    exit 1
+  fi
+  if [ "$dirname" != "$prefix" ]; then
+    mv "$dirname" "$prefix"
+  fi
+  popd
+}
+
+function getrepo() {
+  local dirname="$1"
+  local url="$2"
+  local rev="$3"
+
+  pushd "$SOURCES_DIR"
+  if [ ! -d "$dirname" ]; then
+    git clone "$url" "$dirname"
+    pushd "$dirname"
+    git checkout "$rev"
+    popd
+  fi
+  popd
+
+  pushd "$SOURCES_DIR/$dirname"
+  local oldrev=`git rev-parse HEAD`
+  if [ "$oldrev" != "$rev" ]; then
+    set +x
+    echo "---------------------------------------------------------------"
+    echo "ERROR: sources/$dirname is on wrong revision"
+    echo "--------------------------------------------------------------"
+    echo "Detected revision: $oldrev"
+    echo "Expected revision: $rev"
+    echo "---------------------------------------------------------------"
+    echo "Please pull/checkout to the correct revision, or delete repo"
+    echo "before proceeding (it will be re-cloned)"
+    exit 1
+  fi
+  popd
+}

fetch_sources.sh

@@ -0,0 +1,28 @@
+#!/bin/bash -eux
+
+source common.sh
+
+#
+# To prevent spurious build failures (due to transient network issues),
+# these external archive files are checked into the repository under
+# sources/, but it is always possible to re-download them with:
+#
+#    $ rm -rf sources
+#    $ ./fetch_sources.sh
+#
+
+getsource "https://downloads.xiph.org/releases/ogg/libogg-1.3.5.tar.gz" 0eb4b4b9420a0f51db142ba3f9c64b333f826532dc0f48c6410ae51f4799b664
+getsource "https://downloads.xiph.org/releases/vorbis/libvorbis-1.3.7.tar.gz" 0e982409a9c3fc82ee06e08205b1355e5c6aa4c36bca58146ef399621b0ce5ab
+getsource "https://www.sqlite.org/src/tarball/698edb77/SQLite-698edb77.tar.gz" b1568dc5d17788b9dd9575ecd224b3f7985b51764bc2f34f4808d851332840ef
+getsource "https://www.openssl.org/source/openssl-1.1.1n.tar.gz" 40dceb51a4f6a5275bde0e6bf20ef4b91bfc32ed57c0552e2e8e15463372b17a
+getsource "https://curl.se/download/curl-7.82.0.tar.bz2" 46d9a0400a33408fd992770b04a44a7434b3036f2e8089ac28b57573d59d371f
+getsource "https://github.com/minetest/minetest/releases/download/5.5.1/minetest-5.5.1-win64.zip" 9be48fb4126312b0d8fd048d1ab06d59b3109f6fe037fcd5ab65b99507e68a5f
+
+# These are never checked into the repo, since they are separate git repos.
+# Be sure to add new entries here to .gitignore
+getrepo zlib "https://github.com/madler/zlib.git" 21767c654d31d2dccdde4330529775c6c5fd5389
+getrepo libjpeg "https://github.com/libjpeg-turbo/libjpeg-turbo.git" 2ee7264d40910f2529690de327988ce0c2276812
+getrepo libpng "https://git.code.sf.net/p/libpng/code" a37d4836519517bdce6cb9d956092321eca3e73b
+getrepo freetype "https://gitlab.freedesktop.org/freetype/freetype.git" a8e4563c3418ed74d39019a6c5e2122d12c8f56f
+getrepo zstd "https://github.com/facebook/zstd.git" e47e674cd09583ff0503f0f6defd6d23d8b718d3
+getrepo webshims "https://github.com/paradust7/webshims.git" 01b29b520fa2c97f9c8f79b4bfd5ae67ecba6e06