ci(aio): Refactored payload size script, add script to track payload (#18517)

PR Close #18517

aio/scripts/_payload-limits.sh

@@ -2,9 +2,13 @@
 
 set -u -e -o pipefail
 
-declare -A limitUncompressed
+declare -A payloadLimits
-limitUncompressed=(["inline"]=1600 ["main"]=525000 ["polyfills"]=38000)
+payloadLimits["aio", "uncompressed", "inline"]=1600
-declare -A limitGzip7
+payloadLimits["aio", "uncompressed", "main"]=525000
-limitGzip7=(["inline"]=1000 ["main"]=127000 ["polyfills"]=12500)
+payloadLimits["aio", "uncompressed", "polyfills"]=38000
-declare -A limitGzip9
+payloadLimits["aio", "gzip7", "inline"]=1000
-limitGzip9=(["inline"]=1000 ["main"]=127000 ["polyfills"]=12500)
+payloadLimits["aio", "gzip7", "main"]=127000
+payloadLimits["aio", "gzip7", "polyfills"]=12500
+payloadLimits["aio", "gzip9", "inline"]=1000
+payloadLimits["aio", "gzip9", "main"]=127000
+payloadLimits["aio", "gzip9", "polyfills"]=12500

aio/scripts/payload.sh

@@ -4,79 +4,10 @@ set -eu -o pipefail
 
 readonly thisDir=$(cd $(dirname $0); pwd)
 readonly parentDir=$(dirname $thisDir)
-readonly PROJECT_NAME="angular-payload-size"
 
+# Track payload size functions
+source ../scripts/ci/payload-size.sh
 source ${thisDir}/_payload-limits.sh
 
-failed=false
+trackPayloadSize "aio" "dist/*.bundle.js" true true
-payloadData=""
-for filename in dist/*.bundle.js; do
-  size=$(stat -c%s "$filename")
-  label=$(echo "$filename" | sed "s/.*\///" | sed "s/\..*//")
-  payloadData="$payloadData\"uncompressed/$label\": $size, "
 
-
-  gzip -7 $filename -c >> "${filename}7.gz"
-  size7=$(stat -c%s "${filename}7.gz")
-  payloadData="$payloadData\"gzip7/$label\": $size7, "
-
-  gzip -9 $filename -c >> "${filename}9.gz"
-  size9=$(stat -c%s "${filename}9.gz")
-  payloadData="$payloadData\"gzip9/$label\": $size9, "
-
-  if [[ $size -gt ${limitUncompressed[$label]} ]]; then
-    failed=true
-    echo "Uncompressed $label size is $size which is greater than ${limitUncompressed[$label]}"
-  elif [[ $size7 -gt ${limitGzip7[$label]} ]]; then
-    failed=true
-    echo "Gzip7 $label size is $size7 which is greater than ${limitGzip7[$label]}"
-  elif [[ $size9 -gt ${limitGzip9[$label]} ]]; then
-    failed=true
-    echo "Gzip9 $label size is $size9 which is greater than ${limitGzip9[$label]}"
-  fi
-done
-
-# Add Timestamp
-timestamp=$(date +%s)
-payloadData="$payloadData\"timestamp\": $timestamp, "
-
-# Add change source: application, dependencies, or 'application+dependencies'
-applicationChanged=false
-dependenciesChanged=false
-if [[ $(git diff --name-only $TRAVIS_COMMIT_RANGE $parentDir | grep -v aio/yarn.lock | grep -v content) ]]; then
-  applicationChanged=true
-fi
-if [[ $(git diff --name-only $TRAVIS_COMMIT_RANGE $parentDir/yarn.lock) ]]; then
-  dependenciesChanged=true
-fi
-
-if $dependenciesChanged && $applicationChanged; then
-  change='application+dependencies'
-elif $dependenciesChanged; then
-  # only yarn.lock changed
-  change='dependencies'
-elif $applicationChanged; then
-  change='application'
-else
-  # Nothing changed in aio/
-  exit 0
-fi
-message=$(echo $TRAVIS_COMMIT_MESSAGE | sed 's/"/\\"/g' | sed 's/\\/\\\\/g')
-payloadData="$payloadData\"change\": \"$change\", \"message\": \"$message\""
-
-payloadData="{${payloadData}}"
-
-echo $payloadData
-
-if [[ "$TRAVIS_PULL_REQUEST" == "false" ]]; then
-  readonly safeBranchName=$(echo $TRAVIS_BRANCH | sed -e 's/\./_/g')
-  readonly dbPath=/payload/aio/$safeBranchName/$TRAVIS_COMMIT
-
-  # WARNING: FIREBASE_TOKEN should NOT be printed.
-  set +x
-  firebase database:update --data "$payloadData" --project $PROJECT_NAME --confirm --token "$ANGULAR_PAYLOAD_FIREBASE_TOKEN" $dbPath
-fi
-
-if [[ $failed = true ]]; then
-  exit 1
-fi

integration/_payload-limits.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -u -e -o pipefail
+
+declare -A payloadLimits
+payloadLimits["hello_world__closure", "uncompressed", "bundle"]=106000
+payloadLimits["hello_world__closure", "gzip7", "bundle"]=35000
+payloadLimits["hello_world__closure", "gzip9", "bundle"]=35000
+
+payloadLimits["cli-hello-world", "uncompressed", "inline"]=1500
+payloadLimits["cli-hello-world", "uncompressed", "main"]=183000
+payloadLimits["cli-hello-world", "uncompressed", "polyfills"]=63000
+payloadLimits["cli-hello-world", "gzip7", "inline"]=900
+payloadLimits["cli-hello-world", "gzip7", "main"]=48000
+payloadLimits["cli-hello-world", "gzip7", "polyfills"]=21000
+payloadLimits["cli-hello-world", "gzip9", "inline"]=900
+payloadLimits["cli-hello-world", "gzip9", "main"]=48000
+payloadLimits["cli-hello-world", "gzip9", "polyfills"]=21000
+

integration/run_tests.sh

@@ -4,6 +4,10 @@ set -e -o pipefail
 
 cd `dirname $0`
 
+# Track payload size functions
+source ../scripts/ci/payload-size.sh
+source ./_payload-limits.sh
+
 # Workaround https://github.com/yarnpkg/yarn/issues/2165
 # Yarn will cache file://dist URIs and not update Angular code
 readonly cache=.yarn_local_cache
@@ -34,7 +38,17 @@ for testDir in $(ls | grep -v node_modules) ; do
     cd $testDir
     # Workaround for https://github.com/yarnpkg/yarn/issues/2256
     rm -f yarn.lock
+    rm -rf dist
     yarn install --cache-folder ../$cache
     yarn test || exit 1
+    # Track payload size for cli-hello-world and hello_world__closure
+    if [[ $testDir == cli-hello-world ]] || [[ $testDir == hello_world__closure ]]; then
+      if [[ $testDir == cli-hello-world ]]; then
+        yarn build
+      fi
+      trackPayloadSize "$testDir" "dist/*.js" true false
+    fi
   )
 done
+
+trackPayloadSize "umd" "../dist/packages-dist/*/bundles/*.umd.min.js" false false

scripts/ci/payload-size.sh

@@ -0,0 +1,120 @@
+#!/usr/bin/env bash
+
+readonly PROJECT_NAME="angular-payload-size"
+
+# Calculate the size of target file uncompressed size, gzip7 size, gzip9 size
+# Write to global variable $payloadData, $filename
+calculateSize() {
+  size["uncompressed"]=$(stat -c%s "$filename")
+  label=$(echo "$filename" | sed "s/.*\///" | sed "s/\..*//")
+  payloadData="$payloadData\"uncompressed/$label\": ${size["uncompressed"]}, "
+
+  gzip -7 $filename -c >> "${filename}7.gz"
+  size["gzip7"]=$(stat -c%s "${filename}7.gz")
+  payloadData="$payloadData\"gzip7/$label\": ${size["gzip7"]}, "
+
+  gzip -9 $filename -c >> "${filename}9.gz"
+  size["gzip9"]=$(stat -c%s "${filename}9.gz")
+  payloadData="$payloadData\"gzip9/$label\": ${size["gzip9"]}, "
+}
+
+# Check whether the file size is under limit
+# Write to global variable $failed
+# Read from global variables $size, $size7, $size9, $label, $limitUncompressed
+checkSize() {
+  for fileType in "uncompressed" "gzip7" "gzip9"; do
+    if [[ ${size[$fileType]} -gt ${payloadLimits[$name, $fileType, $label]} ]]; then
+      failed=true
+      echo "$fileType $label size is ${size[$fileType]} which is greater than ${payloadLimits[$name, $fileType, $label]}"
+    fi
+  done
+}
+
+# Write timestamp to global variable $payloadData
+addTimestamp() {
+  # Add Timestamp
+  timestamp=$(date +%s)
+  payloadData="$payloadData\"timestamp\": $timestamp, "
+}
+
+# Write travis commit message to global variable $payloadData
+addMessage() {
+  message=$(git log --oneline $TRAVIS_COMMIT_RANGE)
+  message=$(echo $message | sed 's/"/\\"/g' | sed 's/\\/\\\\/g')
+  payloadData="$payloadData\"message\": \"$message\""
+}
+
+# Add change source: application, dependencies, or 'application+dependencies'
+# Read from global variables $parentDir
+# Update the change source to global variable $payloadData
+addChange() {
+  yarnChanged=false
+  allChangedFiles=$(git diff --name-only $TRAVIS_COMMIT_RANGE $parentDir | wc -l)
+  allChangedFileNames=$(git diff --name-only $TRAVIS_COMMIT_RANGE $parentDir)
+
+  if [[ $allChangedFileNames == *"yarn.lock"* ]]; then
+    yarnChanged=true
+  fi
+
+  if [[ $allChangedFiles -eq 1 ]] && [[ "$yarnChanged" = true ]]; then
+    # only yarn.lock changed
+    change='dependencies'
+  elif [[ $allChangedFiles -gt 1 ]] && [[ "$yarnChanged" = true ]]; then
+    change='application+dependencies'
+  elif [[ $allChangedFiles -gt 0 ]]; then
+    change='application'
+  else
+    # Nothing changed in aio/
+    exit 0
+  fi
+  payloadData="$payloadData\"change\": \"$change\", "
+}
+
+# Upload data to firebase database if it's commit, print out data for pull
+# requests
+uploadData() {
+  name="$1"
+  payloadData="{${payloadData}}"
+
+  echo The data for $name is:
+  echo $payloadData
+
+  if [[ "$TRAVIS_PULL_REQUEST" == "false" ]]; then
+    readonly safeBranchName=$(echo $TRAVIS_BRANCH | sed -e 's/\./_/g')
+    readonly dbPath=/payload/$name/$safeBranchName/$TRAVIS_COMMIT
+
+    # WARNING: FIREBASE_TOKEN should NOT be printed.
+    set +x
+    firebase database:update --data "$payloadData" --project $PROJECT_NAME --confirm --token "$ANGULAR_PAYLOAD_FIREBASE_TOKEN" $dbPath
+  fi
+}
+
+# Track payload size, $1 is the name in database, $2 is the file path
+# $3 is whether we check the payload size and fail the test if the size exceeds
+# limit, $4 is whether record the type of changes: true | false
+trackPayloadSize() {
+  name="$1"
+  path="$2"
+  checkSize="$3"
+  trackChange=$4
+
+  payloadData=""
+  failed=false
+  for filename in $path; do
+    declare -A size
+    calculateSize
+    if [[ $checkSize = true ]]; then
+      checkSize
+    fi
+  done
+  addTimestamp
+  if [[ $trackChange = true ]]; then
+    addChange
+  fi
+  addMessage
+  uploadData $name
+  if [[ $failed = true ]]; then
+    echo exit 1
+    exit 1
+  fi
+}