From bc2ae935c388c2b7e4ef71925890a4546f7bf2c6 Mon Sep 17 00:00:00 2001
From: Tobias Faller <faller@endiio.com>
Date: Sat, 10 Jul 2021 17:22:56 +0200
Subject: [PATCH] Added python pip and pip packages support

---
 src/minimal_overlay/bundles/python/.config    | 22 ++++++++++++
 src/minimal_overlay/bundles/python/01_get.sh  | 35 ++++++++++++++++---
 .../{02_build.sh => 02_build_python.sh}       |  9 +----
 .../bundles/python/03_build_pip.sh            | 17 +++++++++
 .../bundles/python/04_build_packages.sh       | 30 ++++++++++++++++
 .../bundles/python/05_install.sh              | 14 ++++++++
 src/minimal_overlay/bundles/python/bundle.sh  |  5 ++-
 7 files changed, 119 insertions(+), 13 deletions(-)
 rename src/minimal_overlay/bundles/python/{02_build.sh => 02_build_python.sh} (73%)
 create mode 100755 src/minimal_overlay/bundles/python/03_build_pip.sh
 create mode 100755 src/minimal_overlay/bundles/python/04_build_packages.sh
 create mode 100755 src/minimal_overlay/bundles/python/05_install.sh

diff --git a/src/minimal_overlay/bundles/python/.config b/src/minimal_overlay/bundles/python/.config
index 69572dd37..19cbc6b9c 100644
--- a/src/minimal_overlay/bundles/python/.config
+++ b/src/minimal_overlay/bundles/python/.config
@@ -4,3 +4,25 @@
 #
 PYTHON_SOURCE_URL=https://www.python.org/ftp/python/3.8.0/Python-3.8.0.tar.xz
 
+# Automatically downloads and installs pip from the given source:
+#
+#   https://pip.pypa.io/en/stable/installing/
+#
+INSTALL_PIP=false
+PIP_SOURCE_URL=https://bootstrap.pypa.io/get-pip.py
+
+# Installs the list of comma separated packages into the python root.
+# The packages are only installed if INSTALL_PIP is set to "true".
+#
+#   Example: PIP_INSTALL_PACKAGES=console-menu,tqdm
+#
+# Disclaimer: This is currently an experimental feature and not supported
+# in any way as each package might require additional libraries that
+# they are built against (e.g. OpenSSL). This might require additional
+# (not yet existing) overlays to be implemented and built first.
+#
+# Known Python package bundle dependencies:
+# - threading: libgcc_s
+# - gzip: zlib
+#
+PIP_INSTALL_PACKAGES=
diff --git a/src/minimal_overlay/bundles/python/01_get.sh b/src/minimal_overlay/bundles/python/01_get.sh
index 3f741a6a4..19aa905e0 100755
--- a/src/minimal_overlay/bundles/python/01_get.sh
+++ b/src/minimal_overlay/bundles/python/01_get.sh
@@ -8,31 +8,58 @@ set -e
 DOWNLOAD_URL=`read_property PYTHON_SOURCE_URL`
 USE_LOCAL_SOURCE=`read_property USE_LOCAL_SOURCE`
 
+INSTALL_PIP=`read_property INSTALL_PIP`
+PIP_DOWNLOAD_URL=`read_property PIP_SOURCE_URL`
+USE_LOCAL_PIP_SOURCE=`read_property USE_LOCAL_SOURCE`
+
 # Grab everything after the last '/' character.
 ARCHIVE_FILE=${DOWNLOAD_URL##*/}
+PIP_FILE=${PIP_DOWNLOAD_URL##*/}
 
 if [ "$USE_LOCAL_SOURCE" = "true" -a ! -f $MAIN_SRC_DIR/source/overlay/$ARCHIVE_FILE  ] ; then
   echo "Source bundle $MAIN_SRC_DIR/source/overlay/$ARCHIVE_FILE is missing and will be downloaded."
   USE_LOCAL_SOURCE="false"
 fi
 
+if [ "$INSTALL_PIP" = "true" ] ; then
+  if [ "$USE_LOCAL_PIP_SOURCE" = "true" -a ! -f $MAIN_SRC_DIR/source/overlay/$PIP_FILE  ] ; then
+    echo "Pip installation $MAIN_SRC_DIR/source/overlay/$PIP_FILE is missing and will be downloaded."
+    USE_LOCAL_PIP_SOURCE="false"
+  fi
+fi
+
 cd $MAIN_SRC_DIR/source/overlay
 
 if [ ! "$USE_LOCAL_SOURCE" = "true" ] ; then
-  # Downloading vim source bundle file. The '-c' option allows the download to resume.
+  # Downloading python source bundle file. The '-c' option allows the download to resume.
   echo "Downloading PYTHON source bundle from $DOWNLOAD_URL"
   wget -c $DOWNLOAD_URL
 else
   echo "Using local PYTHON source bundle $MAIN_SRC_DIR/source/overlay/$ARCHIVE_FILE"
 fi
 
-# Delete folder with previously extracted vim.
+if [ "$INSTALL_PIP" = "true" ] ; then
+  if [ ! "$USE_LOCAL_PIP_SOURCE" = "true" ] ; then
+    # Downloading pip source bundle file. The '-c' option allows the download to resume.
+    echo "Downloading PIP source bundle from $PIP_DOWNLOAD_URL"
+    wget -c $PIP_DOWNLOAD_URL
+  else
+    echo "Using local PIP source bundle $MAIN_SRC_DIR/source/overlay/$PIP_FILE"
+  fi
+fi
+
+# Delete folder with previously extracted python.
 echo "Removing PYTHON work area. This may take a while."
 rm -rf $WORK_DIR/overlay/$BUNDLE_NAME
 mkdir $WORK_DIR/overlay/$BUNDLE_NAME
 
-# Extract vim to folder 'work/overlay/vim'.
-# Full path will be something like 'work/overlay/vim/vim-8.0.1298'.
+# Extract python to folder 'work/overlay/python'.
+# Full path will be something like 'work/overlay/python/Python-3.8.0'.
 tar -xvf $ARCHIVE_FILE -C $WORK_DIR/overlay/$BUNDLE_NAME
 
+if [ "$INSTALL_PIP" = "true" ] ; then
+  # Copy the pip installation script
+  cp $PIP_FILE $WORK_DIR/overlay/$BUNDLE_NAME/get-pip.py
+fi
+
 cd $SRC_DIR
diff --git a/src/minimal_overlay/bundles/python/02_build.sh b/src/minimal_overlay/bundles/python/02_build_python.sh
similarity index 73%
rename from src/minimal_overlay/bundles/python/02_build.sh
rename to src/minimal_overlay/bundles/python/02_build_python.sh
index 5fd90d588..ea828837f 100755
--- a/src/minimal_overlay/bundles/python/02_build.sh
+++ b/src/minimal_overlay/bundles/python/02_build_python.sh
@@ -28,18 +28,11 @@ make -j $NUM_JOBS
 echo "Installing '$BUNDLE_NAME'."
 make -j $NUM_JOBS install DESTDIR=$DEST_DIR
 
-echo "Generating '$BUNDLE_NAME'."
+#echo "Generating '$BUNDLE_NAME'."
 
 #echo "Reducing '$BUNDLE_NAME' size."
 #set +e
 #strip -g $DEST_DIR/usr/bin/*
 #set -e
 
-# With '--remove-destination' all possibly existing soft links in
-# '$OVERLAY_ROOTFS' will be overwritten correctly.
-cp -r --remove-destination $DEST_DIR/* \
-  $OVERLAY_ROOTFS
-
-echo "Bundle '$BUNDLE_NAME' has been installed."
-
 cd $SRC_DIR
diff --git a/src/minimal_overlay/bundles/python/03_build_pip.sh b/src/minimal_overlay/bundles/python/03_build_pip.sh
new file mode 100755
index 000000000..2000663ad
--- /dev/null
+++ b/src/minimal_overlay/bundles/python/03_build_pip.sh
@@ -0,0 +1,17 @@
+#!/bin/sh
+
+set -e
+
+. ../../common.sh
+
+cd $WORK_DIR/overlay/$BUNDLE_NAME
+
+# Read the common configuration properties.
+INSTALL_PIP=`read_property INSTALL_PIP`
+
+if [ "$INSTALL_PIP" = "true" ] ; then
+  echo "Installing pip"
+  $DEST_DIR/usr/bin/python3 get-pip.py
+fi
+
+cd $SRC_DIR
diff --git a/src/minimal_overlay/bundles/python/04_build_packages.sh b/src/minimal_overlay/bundles/python/04_build_packages.sh
new file mode 100755
index 000000000..ea15edc79
--- /dev/null
+++ b/src/minimal_overlay/bundles/python/04_build_packages.sh
@@ -0,0 +1,30 @@
+#!/bin/sh
+
+set -e
+
+. ../../common.sh
+
+cd $WORK_DIR/overlay/$BUNDLE_NAME
+
+# Read the common configuration properties.
+INSTALL_PIP=`read_property INSTALL_PIP`
+PIP_PACKAGES=`read_property PIP_INSTALL_PACKAGES`
+
+if [ "$INSTALL_PIP" = "true" ] ; then
+  echo "Installing pip packages."
+  for package in ${PIP_PACKAGES//,/ }
+  do 
+    echo "Installing package '$package'."
+
+    CFLAGS="$CFLAGS" \
+	CXXFLAGS="$CXXFLAGS" \
+	$DEST_DIR/usr/bin/python3 -m pip install --force-reinstall \
+      --global-option=build_ext --global-option="--include-dirs=$OVERLAY_ROOTFS/include:$OVERLAY_ROOTFS/usr/include" \
+      --global-option=build_ext --global-option="--library-dirs=$OVERLAY_ROOTFS/lib:$OVERLAY_ROOTFS/usr/lib" \
+      --global-option=build_ext --global-option="--plat-name=linux-x86_64" \
+      --global-option=build_ext --global-option="--force" \
+      $package
+  done
+fi
+
+cd $SRC_DIR
diff --git a/src/minimal_overlay/bundles/python/05_install.sh b/src/minimal_overlay/bundles/python/05_install.sh
new file mode 100755
index 000000000..76f139c8a
--- /dev/null
+++ b/src/minimal_overlay/bundles/python/05_install.sh
@@ -0,0 +1,14 @@
+#!/bin/sh
+
+set -e
+
+. ../../common.sh
+
+# With '--remove-destination' all possibly existing soft links in
+# '$OVERLAY_ROOTFS' will be overwritten correctly.
+cp -r --remove-destination $DEST_DIR/* \
+  $OVERLAY_ROOTFS
+
+echo "Bundle '$BUNDLE_NAME' has been installed."
+
+cd $SRC_DIR
diff --git a/src/minimal_overlay/bundles/python/bundle.sh b/src/minimal_overlay/bundles/python/bundle.sh
index e132bc25f..084f9858f 100755
--- a/src/minimal_overlay/bundles/python/bundle.sh
+++ b/src/minimal_overlay/bundles/python/bundle.sh
@@ -5,6 +5,9 @@ set -e
 . ../../common.sh
 
 ./01_get.sh
-./02_build.sh
+./02_build_python.sh
+./03_build_pip.sh
+./04_build_packages.sh
+./05_install.sh
 
 cd $SRC_DIR