Merge remote-tracking branch 'upstream/main' into experimental-distributed-delete

Author: ScottTodd

.github/workflows/ci.yml

@@ -198,39 +198,15 @@ jobs:
 
   ##############################################################################
 
-  # Depends on all the other jobs to provide a single anchor that indicates the
-  # final status. Status reporting will become more sophisticated in the future
-  # and we can hopefully avoid the need to explicitly list every single job...
-  summary:
-    # Even if you have an explicit if condition, you still need to override
-    # GitHub's default behavior of not running if any dependencies failed.
+  # Aggregate job status and alerting on failures.
+  ci_summary:
     if: always()
-    runs-on: ubuntu-20.04
     needs:
       - setup
       - runtime
       - runtime_small
       - runtime_tracing
-    steps:
-      - name: "Checking out repository"
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-      - name: Getting failed jobs
-        id: failed_jobs
-        run: |
-          echo '${{ toJson(needs) }}'
-          FAILED_JOBS="$(echo '${{ toJson(needs) }}' \
-            | jq --raw-output \
-            'map_values(select(.result!="success" and .result!="skipped")) | keys | join(",")' \
-          )"
-          echo "failed-jobs=${FAILED_JOBS}" >> $GITHUB_OUTPUT
-          if [[ "${FAILED_JOBS}" != "" ]]; then
-            echo "The following jobs failed: ${FAILED_JOBS}"
-            exit 1
-          fi
-      - name: Posting to Discord
-        uses: sarisia/actions-status-discord@ce8cc68e4e626000136b3c702d049a154243e490 # v1.14.7
-        if: failure() && github.ref_name == 'main'
-        with:
-          webhook: ${{ secrets.DISCORD_WEBHOOK }}
-          description: "The following jobs failed: ${{ steps.failed_jobs.outputs.failed-jobs }}"
-          url: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}"
+    uses: ./.github/workflows/workflow_summary.yml
+    secrets: inherit
+    with:
+      jobs-json: ${{ toJson(needs) }}

.github/workflows/pkgci.yml

@@ -104,10 +104,11 @@ jobs:
     if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'test_onnx')
     uses: ./.github/workflows/pkgci_test_onnx.yml
 
+  # TODO(https://github.com/iree-org/iree-test-suites/issues/56): re-enable when git LFS quota is available
   test_sharktank:
     name: Test Sharktank
     needs: [setup, build_packages]
-    if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'test_sharktank')
+    if: false && contains(fromJson(needs.setup.outputs.enabled-jobs), 'test_sharktank')
     uses: ./.github/workflows/pkgci_test_sharktank.yml
 
   test_tensorflow:
@@ -121,3 +122,28 @@ jobs:
     needs: [setup, build_packages]
     if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'test_pjrt')
     uses: ./.github/workflows/pkgci_test_pjrt.yml
+
+  ##############################################################################
+
+  # Aggregate job status and alerting on failures.
+  pkgci_summary:
+    if: always()
+    needs:
+      - setup
+      - build_packages
+      - unit_test
+      - regression_test
+      - test_amd_mi250
+      - test_amd_mi300
+      - test_amd_w7900
+      # - test_nvidia_t4
+      - test_android
+      - test_riscv64
+      - test_onnx
+      - test_sharktank
+      - test_tensorflow
+      - test_pjrt
+    uses: ./.github/workflows/workflow_summary.yml
+    secrets: inherit
+    with:
+      jobs-json: ${{ toJson(needs) }}

.github/workflows/workflow_summary.yml

@@ -0,0 +1,63 @@
+# Copyright 2024 The IREE Authors
+#
+# Licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+# Checks the result status of each job provided by 'jobs-json' and sends an
+# alert if at least one job failed.
+#
+# Usage:
+#   ```yml
+#   jobs:
+#      job_1:
+#          ...
+#      job_2:
+#          ...
+#      my_summary:
+#        if: always()
+#        needs:
+#          - job_1
+#          - job_2
+#        uses: ./.github/workflows/workflow_summary.yml
+#        secrets: inherit
+#        with:
+#          jobs-json: ${{ toJson(needs) }}
+#   ```
+
+name: Workflow Summary
+
+on:
+  workflow_call:
+    inputs:
+        jobs-json:
+          type: string
+          description: The output of `toJson(needs)`
+
+permissions:
+  contents: read
+
+jobs:
+  summary:
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Getting failed jobs
+        id: failed_jobs
+        run: |
+          echo '${{ inputs.jobs-json }}'
+          FAILED_JOBS="$(echo '${{ inputs.jobs-json }}' \
+            | jq --raw-output \
+            'map_values(select(.result!="success" and .result!="skipped")) | keys | join(",")' \
+          )"
+          echo "failed-jobs=${FAILED_JOBS}" >> $GITHUB_OUTPUT
+          if [[ "${FAILED_JOBS}" != "" ]]; then
+            echo "The following jobs failed: ${FAILED_JOBS}"
+            exit 1
+          fi
+      - name: Posting to Discord
+        uses: sarisia/actions-status-discord@ce8cc68e4e626000136b3c702d049a154243e490 # v1.14.7
+        if: failure() && github.ref_name == 'main' && github.repository_owner == 'iree-org'
+        with:
+          webhook: ${{ secrets.DISCORD_WEBHOOK }}
+          description: "The following jobs failed: ${{ steps.failed_jobs.outputs.failed-jobs }}"
+          url: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}"

build_tools/github_actions/configure_ci.py

@@ -39,8 +39,8 @@
 import fnmatch
 import json
 import os
-import re
 import pathlib
+import re
 import string
 import subprocess
 import sys
@@ -111,7 +111,12 @@ def contains(cls, val):
 RUNNER_ENV_DEFAULT = "prod"
 RUNNER_ENV_OPTIONS = [RUNNER_ENV_DEFAULT, "testing"]
 
-CONTROL_JOBS = frozenset(["setup", "summary"])
+CONTROL_JOB_REGEXES = frozenset(
+    [
+        re.compile("setup"),
+        re.compile(".*summary"),
+    ]
+)
 
 # Jobs to run only on postsubmit by default.
 # They may also run on presubmit only under certain conditions.
@@ -380,7 +385,8 @@ def parse_jobs_from_workflow_file(workflow_file: pathlib.Path) -> Set[str]:
 
     workflow = yaml.load(workflow_file.read_text(), Loader=yaml.SafeLoader)
     all_jobs = set(workflow["jobs"].keys())
-    all_jobs -= CONTROL_JOBS
+    for regex in CONTROL_JOB_REGEXES:
+        all_jobs = {j for j in all_jobs if not regex.match(j)}
 
     if ALL_KEY in all_jobs:
         raise ValueError(f"Workflow has job with reserved name '{ALL_KEY}'")

compiler/src/iree/compiler/Codegen/Common/EncodingUtils.cpp

@@ -100,16 +100,13 @@ MaterializeEncodingTypeConverter::MaterializeEncodingTypeConverter(
     // itself.
     RankedTensorType tensorType =
         transposeNarrowN ? transposeIfNarrowNResult(type) : type;
-    FailureOr<MaterializeEncodingInfo> maybeEncodingInfo =
-        getEncodingInfo(tensorType);
-    if (failed(maybeEncodingInfo) ||
-        IREE::Codegen::isIdentityLayout(maybeEncodingInfo.value())) {
+    MaterializeEncodingInfo encodingInfo = getEncodingInfo(tensorType);
+    if (IREE::Codegen::isIdentityLayout(encodingInfo)) {
       return dropEncoding(type);
     }
-    auto encodingInfo = *maybeEncodingInfo;
     auto packedType = cast<RankedTensorType>(tensor::PackOp::inferPackedType(
-        tensorType, maybeEncodingInfo->innerTileSizes,
-        maybeEncodingInfo->innerDimsPos, maybeEncodingInfo->outerDimsPerm));
+        tensorType, encodingInfo.innerTileSizes, encodingInfo.innerDimsPos,
+        encodingInfo.outerDimsPerm));
 
     // There is no swizzle, we are already done. Typically the case on CPU.
     if (!encodingInfo.swizzle) {

compiler/src/iree/compiler/Codegen/Common/EncodingUtils.h

@@ -42,7 +42,7 @@ class MaterializeEncodingTypeConverter : public TypeConverter {
     return layoutAttr;
   }
 
-  FailureOr<IREE::Codegen::MaterializeEncodingInfo>
+  IREE::Codegen::MaterializeEncodingInfo
   getEncodingInfo(RankedTensorType type) const {
     return layoutAttr.getEncodingInfo(type);
   }

compiler/src/iree/compiler/Codegen/Common/GPU/GPUMaterializeEncoding.cpp

@@ -108,13 +108,9 @@ struct GPUSetEncodingOpLoweringConversion
       return success();
     }
 
-    FailureOr<MaterializeEncodingInfo> maybeEncodingInfo =
+    MaterializeEncodingInfo encodingInfo =
         converter->getEncodingInfo(encodingOp.getResultType());
-    if (failed(maybeEncodingInfo)) {
-      return rewriter.notifyMatchFailure(encodingOp,
-                                         "unhandled result encoding");
-    }
-    if (!maybeEncodingInfo->swizzle) {
+    if (!encodingInfo.swizzle) {
       rewriter.replaceOp(encodingOp, packedValue.value());
       return success();
     }
@@ -128,18 +124,18 @@ struct GPUSetEncodingOpLoweringConversion
             .getShape()
             .take_front(origRank));
     expandShapeShape.append(
-        getExpandedTileShape(maybeEncodingInfo->swizzle->expandShape));
+        getExpandedTileShape(encodingInfo.swizzle->expandShape));
     RankedTensorType expandShapeType =
         encodingOp.getSourceType().clone(expandShapeShape);
 
-    SmallVector<ReassociationIndices> reassociation = getReassociationIndices(
-        origRank, maybeEncodingInfo->swizzle->expandShape);
+    SmallVector<ReassociationIndices> reassociation =
+        getReassociationIndices(origRank, encodingInfo.swizzle->expandShape);
     auto expandShapeOp = rewriter.create<tensor::ExpandShapeOp>(
         loc, expandShapeType, packedValue.value(), reassociation);
 
     SmallVector<int64_t> transposePerm =
         llvm::to_vector(llvm::seq<int64_t>(0, origRank));
-    for (auto perm : maybeEncodingInfo->swizzle->permutation) {
+    for (auto perm : encodingInfo.swizzle->permutation) {
       transposePerm.push_back(origRank + perm);
     }
     SmallVector<OpFoldResult> transposeResultDims =
@@ -168,9 +164,9 @@ struct GPUUnsetEncodingOpLoweringConversion
     auto converter = static_cast<const MaterializeEncodingTypeConverter *>(
         getTypeConverter());
 
-    FailureOr<MaterializeEncodingInfo> maybeEncodingInfo =
+    MaterializeEncodingInfo encodingInfo =
         converter->getEncodingInfo(unsetEncodingOp.getSource().getType());
-    if (failed(maybeEncodingInfo)) {
+    if (IREE::Codegen::isIdentityLayout(encodingInfo)) {
       Type targetType =
           getTypeConverter()->convertType(unsetEncodingOp.getSourceType());
       Value result = rewriter.createOrFold<tensor::CastOp>(
@@ -181,35 +177,34 @@ struct GPUUnsetEncodingOpLoweringConversion
 
     Location loc = unsetEncodingOp.getLoc();
     Value unpackSrc = adaptor.getSource();
-    if (maybeEncodingInfo->swizzle) {
+    if (encodingInfo.swizzle) {
       int targetRank = unsetEncodingOp.getResultType().getRank();
       auto srcConvertedType =
           cast<RankedTensorType>(adaptor.getSource().getType());
       SmallVector<OpFoldResult> emptyShape =
           tensor::getMixedSizes(rewriter, loc, adaptor.getSource());
       emptyShape.resize(targetRank);
-      for (auto i :
-           getExpandedTileShape(maybeEncodingInfo->swizzle->expandShape)) {
+      for (auto i : getExpandedTileShape(encodingInfo.swizzle->expandShape)) {
         emptyShape.push_back(rewriter.getIndexAttr(i));
       }
       auto emptyTensor = rewriter.create<tensor::EmptyOp>(
           loc, emptyShape, unsetEncodingOp.getSourceType().getElementType());
 
       SmallVector<int64_t> transposePerm =
           llvm::to_vector(llvm::seq<int64_t>(0, targetRank));
-      for (auto perm : maybeEncodingInfo->swizzle->permutation) {
+      for (auto perm : encodingInfo.swizzle->permutation) {
         transposePerm.push_back(targetRank + perm);
       }
       auto invertedTransposePerm = invertPermutationVector(transposePerm);
       auto transposeOp = rewriter.create<linalg::TransposeOp>(
           loc, adaptor.getSource(), emptyTensor, invertedTransposePerm);
 
       SmallVector<ReassociationIndices> reassociation = getReassociationIndices(
-          targetRank, maybeEncodingInfo->swizzle->expandShape);
+          targetRank, encodingInfo.swizzle->expandShape);
       SmallVector<int64_t> unpackSrcShape(
           srcConvertedType.getShape().take_front(targetRank));
-      unpackSrcShape.append(maybeEncodingInfo->innerTileSizes.begin(),
-                            maybeEncodingInfo->innerTileSizes.end());
+      unpackSrcShape.append(encodingInfo.innerTileSizes.begin(),
+                            encodingInfo.innerTileSizes.end());
       RankedTensorType unpackSrcType =
           unsetEncodingOp.getResultType().clone(unpackSrcShape);
       unpackSrc = rewriter.create<tensor::CollapseShapeOp>(