mirror of
https://github.com/zama-ai/concrete.git
synced 2026-02-08 19:44:57 -05:00
c8c969773e
This commit rebases the compiler onto commit 465ee9bfb26d from
llvm-project with locally maintained patches on top, i.e.:
* 5d8669d669ee: Fix the element alignment (size) for memrefCopy
* 4239163ea337: fix: Do not fold the memref.subview if the offset are
!= 0 and strides != 1
* 72c5decfcc21: remove github stuff from llvm
* 8d0ce8f9eca1: Support arbitrary element types in named operations
via attributes
* 94f64805c38c: Copy attributes of scf.for on bufferization and make
it an allocation hoisting barrier
Main upstream changes from llvm-project that required modification of
concretecompiler:
* Switch to C++17
* Various changes in the interfaces for linalg named operations
* Transition from `llvm::Optional` to `std::optional`
* Use of enums instead of string values for iterator types in linalg
* Changed default naming convention of getter methods in
ODS-generated operation classes from `some_value()` to
`getSomeValue()`
* Renaming of Arithmetic dialect to Arith
* Refactoring of side effect interfaces (i.e., renaming from
`NoSideEffect` to `Pure`)
* Re-design of the data flow analysis framework
* Refactoring of build targets for Python bindings
* Refactoring of array attributes with integer values
* Renaming of `linalg.init_tensor` to `tensor.empty`
* Emission of `linalg.map` operations in bufferization of the Tensor
dialect requiring another linalg conversion pass and registration
of the bufferization op interfaces for linalg operations
* Refactoring of the one-shot bufferizer
* Necessity to run the expand-strided-metadata, affine-to-std and
finalize-memref-to-llvm passes before converson to the LLVM
dialect
* Renaming of `BlockAndValueMapping` to `IRMapping`
* Changes in the build function of `LLVM::CallOp`
* Refactoring of the construction of `llvm::ArrayRef` and
`llvm::MutableArrayRef` (direct invocation of constructor instead
of builder functions for some cases)
* New naming conventions for generated SSA values requiring rewrite
of some check tests
* Refactoring of `mlir::LLVM::lookupOrCreateMallocFn()`
* Interface changes in generated type parsers
* New dependencies for to mlir_float16_utils and
MLIRSparseTensorRuntime for the runtime
* Overhaul of MLIR-c deleting `mlir-c/Registration.h`
* Deletion of library MLIRLinalgToSPIRV
* Deletion of library MLIRLinalgAnalysis
* Deletion of library MLIRMemRefUtils
* Deletion of library MLIRQuantTransforms
* Deletion of library MLIRVectorToROCDL
343 lines
11 KiB
C++
343 lines
11 KiB
C++
// Part of the Concrete Compiler Project, under the BSD3 License with Zama
|
|
// Exceptions. See
|
|
// https://github.com/zama-ai/concrete-compiler-internal/blob/main/LICENSE.txt
|
|
// for license information.
|
|
|
|
#include "mlir/Dialect/Bufferization/IR/Bufferization.h"
|
|
|
|
#include "llvm/ADT/STLExtras.h"
|
|
#include "llvm/ADT/SmallVector.h"
|
|
#include "llvm/Support/Debug.h"
|
|
#include <mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h>
|
|
#include <mlir/Dialect/Func/IR/FuncOps.h>
|
|
#include <mlir/IR/BuiltinTypes.h>
|
|
#include <mlir/Transforms/RegionUtils.h>
|
|
|
|
#include "concretelang/Dialect/RT/IR/RTDialect.h"
|
|
#include "concretelang/Dialect/RT/IR/RTOps.h"
|
|
|
|
using namespace mlir;
|
|
using namespace mlir::bufferization;
|
|
using namespace mlir::concretelang::RT;
|
|
// using namespace mlir::tensor;
|
|
|
|
namespace {
|
|
struct DerefWorkFunctionArgumentPtrPlaceholderOpBufferizationInterface
|
|
: public BufferizableOpInterface::ExternalModel<
|
|
DerefWorkFunctionArgumentPtrPlaceholderOpBufferizationInterface,
|
|
DerefWorkFunctionArgumentPtrPlaceholderOp> {
|
|
bool bufferizesToMemoryRead(Operation *op, OpOperand &opOperand,
|
|
const AnalysisState &state) const {
|
|
return false;
|
|
}
|
|
|
|
bool bufferizesToMemoryWrite(Operation *op, OpOperand &opOperand,
|
|
const AnalysisState &state) const {
|
|
return false;
|
|
}
|
|
|
|
AliasingOpResultList getAliasingOpResults(Operation *op, OpOperand &opOperand,
|
|
const AnalysisState &state) const {
|
|
return {};
|
|
}
|
|
|
|
BufferRelation bufferRelation(Operation *op, OpResult opResult,
|
|
const AnalysisState &state) const {
|
|
return BufferRelation::Unknown;
|
|
}
|
|
|
|
LogicalResult bufferize(Operation *bop, RewriterBase &rewriter,
|
|
const BufferizationOptions &options) const {
|
|
DerefWorkFunctionArgumentPtrPlaceholderOp op =
|
|
cast<DerefWorkFunctionArgumentPtrPlaceholderOp>(bop);
|
|
|
|
auto isTensorType = [](Type t) { return t.isa<TensorType>(); };
|
|
bool hasTensorResult = llvm::any_of(op->getResultTypes(), isTensorType);
|
|
bool hasTensorOperand = llvm::any_of(op->getOperandTypes(), isTensorType);
|
|
|
|
if (!hasTensorResult && !hasTensorOperand)
|
|
return success();
|
|
|
|
SmallVector<mlir::Value, 2> newOperands;
|
|
|
|
for (OpOperand &opOperand : op->getOpOperands()) {
|
|
Value oldOperandValue = opOperand.get();
|
|
|
|
if (oldOperandValue.getType().isa<TensorType>()) {
|
|
FailureOr<Value> bufferOrErr =
|
|
bufferization::getBuffer(rewriter, opOperand.get(), options);
|
|
|
|
if (failed(bufferOrErr))
|
|
return failure();
|
|
|
|
Value buffer = bufferOrErr.value();
|
|
newOperands.push_back(buffer);
|
|
} else {
|
|
newOperands.push_back(opOperand.get());
|
|
}
|
|
}
|
|
|
|
SmallVector<mlir::Type, 2> newResultTypes;
|
|
|
|
for (OpResult res : op->getResults()) {
|
|
if (TensorType t = res.getType().dyn_cast<TensorType>()) {
|
|
BaseMemRefType memrefType = getMemRefType(res, options);
|
|
newResultTypes.push_back(memrefType);
|
|
} else {
|
|
newResultTypes.push_back(res.getType());
|
|
}
|
|
}
|
|
|
|
rewriter.setInsertionPoint(op);
|
|
DerefWorkFunctionArgumentPtrPlaceholderOp newOp =
|
|
rewriter.create<DerefWorkFunctionArgumentPtrPlaceholderOp>(
|
|
op.getLoc(), newResultTypes, newOperands);
|
|
|
|
replaceOpWithBufferizedValues(rewriter, op, newOp->getResults());
|
|
|
|
return success();
|
|
}
|
|
};
|
|
|
|
struct MakeReadyFutureOpBufferizationInterface
|
|
: public BufferizableOpInterface::ExternalModel<
|
|
MakeReadyFutureOpBufferizationInterface, MakeReadyFutureOp> {
|
|
bool bufferizesToMemoryRead(Operation *op, OpOperand &opOperand,
|
|
const AnalysisState &state) const {
|
|
return false;
|
|
}
|
|
|
|
bool bufferizesToMemoryWrite(Operation *op, OpOperand &opOperand,
|
|
const AnalysisState &state) const {
|
|
return false;
|
|
}
|
|
|
|
AliasingOpResultList getAliasingOpResults(Operation *op, OpOperand &opOperand,
|
|
const AnalysisState &state) const {
|
|
return {};
|
|
}
|
|
|
|
BufferRelation bufferRelation(Operation *op, OpResult opResult,
|
|
const AnalysisState &state) const {
|
|
return BufferRelation::Unknown;
|
|
}
|
|
|
|
LogicalResult bufferize(Operation *bop, RewriterBase &rewriter,
|
|
const BufferizationOptions &options) const {
|
|
MakeReadyFutureOp op = cast<MakeReadyFutureOp>(bop);
|
|
|
|
auto isTensorType = [](Type t) { return t.isa<TensorType>(); };
|
|
bool hasTensorResult = llvm::any_of(op->getResultTypes(), isTensorType);
|
|
bool hasTensorOperand = llvm::any_of(op->getOperandTypes(), isTensorType);
|
|
|
|
if (!hasTensorResult && !hasTensorOperand)
|
|
return success();
|
|
|
|
SmallVector<mlir::Value, 2> newOperands;
|
|
|
|
for (OpOperand &opOperand : op->getOpOperands()) {
|
|
Value oldOperandValue = opOperand.get();
|
|
|
|
if (oldOperandValue.getType().isa<TensorType>()) {
|
|
FailureOr<Value> bufferOrErr =
|
|
bufferization::getBuffer(rewriter, opOperand.get(), options);
|
|
|
|
if (failed(bufferOrErr))
|
|
return failure();
|
|
|
|
Value buffer = bufferOrErr.value();
|
|
newOperands.push_back(buffer);
|
|
} else {
|
|
newOperands.push_back(opOperand.get());
|
|
}
|
|
}
|
|
|
|
SmallVector<mlir::Type, 2> newResultTypes;
|
|
|
|
for (OpResult res : op->getResults()) {
|
|
if (TensorType t = res.getType().dyn_cast<TensorType>()) {
|
|
BaseMemRefType memrefType = getMemRefType(res, options);
|
|
newResultTypes.push_back(memrefType);
|
|
} else {
|
|
newResultTypes.push_back(res.getType());
|
|
}
|
|
}
|
|
|
|
rewriter.setInsertionPoint(op);
|
|
MakeReadyFutureOp newOp = rewriter.create<MakeReadyFutureOp>(
|
|
op.getLoc(), newResultTypes, newOperands);
|
|
|
|
replaceOpWithBufferizedValues(rewriter, op, newOp->getResults());
|
|
|
|
return success();
|
|
}
|
|
};
|
|
|
|
struct WorkFunctionReturnOpBufferizationInterface
|
|
: public BufferizableOpInterface::ExternalModel<
|
|
WorkFunctionReturnOpBufferizationInterface, WorkFunctionReturnOp> {
|
|
bool bufferizesToMemoryRead(Operation *op, OpOperand &opOperand,
|
|
const AnalysisState &state) const {
|
|
return false;
|
|
}
|
|
|
|
bool bufferizesToMemoryWrite(Operation *op, OpOperand &opOperand,
|
|
const AnalysisState &state) const {
|
|
return false;
|
|
}
|
|
|
|
AliasingOpResultList getAliasingOpResults(Operation *op, OpOperand &opOperand,
|
|
const AnalysisState &state) const {
|
|
return {};
|
|
}
|
|
|
|
BufferRelation bufferRelation(Operation *op, OpResult opResult,
|
|
const AnalysisState &state) const {
|
|
return BufferRelation::Unknown;
|
|
}
|
|
|
|
LogicalResult bufferize(Operation *bop, RewriterBase &rewriter,
|
|
const BufferizationOptions &options) const {
|
|
WorkFunctionReturnOp op = cast<WorkFunctionReturnOp>(bop);
|
|
|
|
auto isTensorType = [](Type t) { return t.isa<TensorType>(); };
|
|
bool hasTensorResult = llvm::any_of(op->getResultTypes(), isTensorType);
|
|
bool hasTensorOperand = llvm::any_of(op->getOperandTypes(), isTensorType);
|
|
|
|
if (!hasTensorResult && !hasTensorOperand)
|
|
return success();
|
|
|
|
SmallVector<mlir::Value, 2> newOperands;
|
|
|
|
for (OpOperand &opOperand : op->getOpOperands()) {
|
|
Value oldOperandValue = opOperand.get();
|
|
|
|
if (oldOperandValue.getType().isa<TensorType>()) {
|
|
FailureOr<Value> bufferOrErr =
|
|
bufferization::getBuffer(rewriter, opOperand.get(), options);
|
|
|
|
if (failed(bufferOrErr))
|
|
return failure();
|
|
|
|
Value buffer = bufferOrErr.value();
|
|
newOperands.push_back(buffer);
|
|
} else {
|
|
newOperands.push_back(opOperand.get());
|
|
}
|
|
}
|
|
|
|
SmallVector<mlir::Type, 2> newResultTypes;
|
|
|
|
for (OpResult res : op->getResults()) {
|
|
if (TensorType t = res.getType().dyn_cast<TensorType>()) {
|
|
BaseMemRefType memrefType = getMemRefType(res, options);
|
|
newResultTypes.push_back(memrefType);
|
|
} else {
|
|
newResultTypes.push_back(res.getType());
|
|
}
|
|
}
|
|
|
|
rewriter.setInsertionPoint(op);
|
|
WorkFunctionReturnOp newOp = rewriter.create<WorkFunctionReturnOp>(
|
|
op.getLoc(), newResultTypes, newOperands);
|
|
|
|
replaceOpWithBufferizedValues(rewriter, op, newOp->getResults());
|
|
|
|
return success();
|
|
}
|
|
};
|
|
|
|
struct AwaitFutureOpBufferizationInterface
|
|
: public BufferizableOpInterface::ExternalModel<
|
|
AwaitFutureOpBufferizationInterface, AwaitFutureOp> {
|
|
bool bufferizesToMemoryRead(Operation *op, OpOperand &opOperand,
|
|
const AnalysisState &state) const {
|
|
return false;
|
|
}
|
|
|
|
bool bufferizesToMemoryWrite(Operation *op, OpOperand &opOperand,
|
|
const AnalysisState &state) const {
|
|
return false;
|
|
}
|
|
|
|
AliasingOpResultList getAliasingOpResults(Operation *op, OpOperand &opOperand,
|
|
const AnalysisState &state) const {
|
|
return {};
|
|
}
|
|
|
|
BufferRelation bufferRelation(Operation *op, OpResult opResult,
|
|
const AnalysisState &state) const {
|
|
return BufferRelation::Unknown;
|
|
}
|
|
|
|
LogicalResult bufferize(Operation *bop, RewriterBase &rewriter,
|
|
const BufferizationOptions &options) const {
|
|
AwaitFutureOp op = cast<AwaitFutureOp>(bop);
|
|
|
|
auto isTensorType = [](Type t) { return t.isa<TensorType>(); };
|
|
bool hasTensorResult = llvm::any_of(op->getResultTypes(), isTensorType);
|
|
bool hasTensorOperand = llvm::any_of(op->getOperandTypes(), isTensorType);
|
|
|
|
if (!hasTensorResult && !hasTensorOperand)
|
|
return success();
|
|
|
|
SmallVector<mlir::Value, 2> newOperands;
|
|
|
|
for (OpOperand &opOperand : op->getOpOperands()) {
|
|
Value oldOperandValue = opOperand.get();
|
|
|
|
if (oldOperandValue.getType().isa<TensorType>()) {
|
|
FailureOr<Value> bufferOrErr =
|
|
bufferization::getBuffer(rewriter, opOperand.get(), options);
|
|
|
|
if (failed(bufferOrErr))
|
|
return failure();
|
|
|
|
Value buffer = bufferOrErr.value();
|
|
newOperands.push_back(buffer);
|
|
} else {
|
|
newOperands.push_back(opOperand.get());
|
|
}
|
|
}
|
|
|
|
SmallVector<mlir::Type, 2> newResultTypes;
|
|
|
|
for (OpResult res : op->getResults()) {
|
|
if (TensorType t = res.getType().dyn_cast<TensorType>()) {
|
|
BaseMemRefType memrefType = getMemRefType(res, options);
|
|
newResultTypes.push_back(memrefType);
|
|
} else {
|
|
newResultTypes.push_back(res.getType());
|
|
}
|
|
}
|
|
|
|
rewriter.setInsertionPoint(op);
|
|
AwaitFutureOp newOp = rewriter.create<AwaitFutureOp>(
|
|
op.getLoc(), newResultTypes, newOperands);
|
|
|
|
replaceOpWithBufferizedValues(rewriter, op, newOp->getResults());
|
|
|
|
return success();
|
|
}
|
|
};
|
|
|
|
} // namespace
|
|
|
|
namespace mlir {
|
|
namespace concretelang {
|
|
namespace RT {
|
|
void registerBufferizableOpInterfaceExternalModels(DialectRegistry ®istry) {
|
|
registry.addExtension(+[](MLIRContext *ctx, RTDialect *dialect) {
|
|
DerefWorkFunctionArgumentPtrPlaceholderOp::attachInterface<
|
|
DerefWorkFunctionArgumentPtrPlaceholderOpBufferizationInterface>(*ctx);
|
|
AwaitFutureOp::attachInterface<AwaitFutureOpBufferizationInterface>(*ctx);
|
|
MakeReadyFutureOp::attachInterface<MakeReadyFutureOpBufferizationInterface>(
|
|
*ctx);
|
|
WorkFunctionReturnOp::attachInterface<
|
|
WorkFunctionReturnOpBufferizationInterface>(*ctx);
|
|
});
|
|
}
|
|
} // namespace RT
|
|
} // namespace concretelang
|
|
} // namespace mlir
|