scripts: adjust relocation.inc to not copy large directories

Some of the git repositories for depedencies can be quite large.
The large files never seem to be related to build (as they would
be too large to be pure go modules).

To make things faster, update our rsync copy to exclude any
directories bigger than 500M, we can adjust the limit or make
it something a recipe can specify in the future, but for now
this helps long build times.

Signed-off-by: Bruce Ashfield <bruce.ashfield@gmail.com>

1 files changed, 29 insertions, 4 deletions

diff --git a/scripts/oe-go-mod-autogen.py b/scripts/oe-go-mod-autogen.py
index 40c964da..7cb101d0 100755
--- a/scripts/oe-go-mod-autogen.py
+++ b/scripts/oe-go-mod-autogen.py
@@ -569,8 +569,7 @@ class GoModTool(object):
         srcrev_name_recorded = []
         # pre styhead releases
         # SRC_URI += "git://%s;name=%s;protocol=https;nobranch=1;destsuffix=${WORKDIR}/${BP}/src/import/vendor.fetch/%s"
-        template = """#       %s %s
-# [1] git ls-remote %s %s
+        template = """# [%s %s] git ls-remote %s %s
 SRCREV_%s = "%s"
 SRC_URI += "git://%s;name=%s;protocol=https;nobranch=1;destsuffix=${GO_SRCURI_DESTSUFFIX}/vendor.fetch/%s"
 
@@ -622,14 +621,40 @@ do_compile:prepend() {
         site_dest=$(echo $s | cut -d: -f1)
         site_source=$(echo $s | cut -d: -f2)
         force_flag=$(echo $s | cut -d: -f3)
+
         mkdir -p vendor.copy/$site_dest
+
+        # create a temporary exclude file
+        exclude_file=$(mktemp)
+
+        find vendor.fetch/$site_source -type d -print0 | \
+        xargs -0 du -sBM 2>/dev/null | \
+        awk '{if ($1+0 > 500) print substr($0, index($0,$2))}' | \
+        sed 's|^vendor.fetch/||' > "$exclude_file"
+
         if [ -n "$force_flag" ]; then
             echo "[INFO] $site_dest: force copying .go files"
             rm -rf vendor.copy/$site_dest
-            rsync -a --exclude='vendor/' --exclude='.git/' vendor.fetch/$site_source/ vendor.copy/$site_dest
+            rsync -a \
+                --exclude='vendor/' \
+                --exclude='.git/' \
+                --exclude-from="$exclude_file" \
+                vendor.fetch/$site_source/ vendor.copy/$site_dest
         else
-            [ -n "$(ls -A vendor.copy/$site_dest/*.go 2> /dev/null)" ] && { echo "[INFO] vendor.fetch/$site_source -> $site_dest: go copy skipped (files present)" ; true ; } || { echo "[INFO] $site_dest: copying .go files" ; rsync -a --exclude='vendor/' --exclude='.git/' vendor.fetch/$site_source/ vendor.copy/$site_dest ; }
+            if [ -n "$(ls -A vendor.copy/$site_dest/*.go 2> /dev/null)" ]; then
+                echo "[INFO] vendor.fetch/$site_source -> $site_dest: go copy skipped (files present)"
+                true
+            else
+                echo "[INFO] $site_dest: copying .go files"
+                rsync -a \
+                    --exclude='vendor/' \
+                    --exclude='.git/' \
+                    --exclude-from="$exclude_file" \
+                    vendor.fetch/$site_source/ vendor.copy/$site_dest
+            fi
         fi
+
+        rm -f "$exclude_file"
     done
 }
 """