BindPython ABI (#2796)

* importing `cpython_bindings.py` when using `BindPython` ABI cpython_bindings.py will contain declarations for all the CPython C API function required * BindPython for no args - no return functions in LLVM backend * BindPython support for args of int and float types * refactored "from ... import ..." AST to ASR code * BindPython support for args of str and bool types * BindPython support for return type of str, bool, integer & real types * refactored python_bind split one `pass_python_bind` function into `generate_body`, `native_to_cpython`, `cpython_to_native`, and `pass_python_bind` * fix for CI * fix for failing test * add integration test for llvm backend * refactor: importing cpython_bindings separate out into function * skip python_bind ASR pass when using C backend * changes according to code review * generating CPython related function declarations in python_bind pass * fix for failing CI * remove use of `PyRun_SimpleString` to set python path some refactoring * clean up unwanted comment * refactored `declare_functions` to asr_utils.cpp * Update src/libasr/pass/python_bind.cpp Co-authored-by: Shaikh Ubaid <[email protected]> * skipping python_bind ASR pass if `--enable-cpython` flag not used * fix related to previous commit --------- Co-authored-by: Shaikh Ubaid <[email protected]>
lcompilers · Aug 12, 2024 · 4b06d70 · 4b06d70
1 parent 9374feb
commit 4b06d70
Show file tree

Hide file tree

Showing 11 changed files with 615 additions and 13 deletions.
diff --git a/integration_tests/CMakeLists.txt b/integration_tests/CMakeLists.txt
@@ -660,7 +660,7 @@ RUN(NAME bindc_05   LABELS llvm c
         EXTRAFILES bindc_05b.c)
 RUN(NAME bindc_06   LABELS llvm c
         EXTRAFILES bindc_06b.c)
-RUN(NAME bindpy_01           LABELS cpython c_py EXTRA_ARGS --enable-cpython NOFAST COPY_TO_BIN bindpy_01_module.py)
+RUN(NAME bindpy_01           LABELS cpython llvm_py c_py EXTRA_ARGS --enable-cpython NOFAST COPY_TO_BIN bindpy_01_module.py)
 RUN(NAME bindpy_02           LABELS cpython c_py EXTRA_ARGS --link-numpy COPY_TO_BIN bindpy_02_module.py)
 RUN(NAME bindpy_03           LABELS cpython c_py EXTRA_ARGS --link-numpy NOFAST COPY_TO_BIN bindpy_03_module.py)
 RUN(NAME bindpy_04           LABELS cpython c_py EXTRA_ARGS --link-numpy NOFAST COPY_TO_BIN bindpy_04_module.py)

diff --git a/integration_tests/bindpy_05.py b/integration_tests/bindpy_05.py
@@ -41,7 +41,7 @@ def PyObject_CallObject(a: CPtr, b: CPtr) -> CPtr:
     pass
 
 @ccall(header="Python.h")
-def PyLong_AsLongLong(a: CPtr) -> i32:
+def PyLong_AsLongLong(a: CPtr) -> i64:
     pass
 
 def my_f():
@@ -62,9 +62,9 @@ def my_f():
     _Py_DecRef(pArgs)
     assert bool(pValue), "Call to my_f failed\n"
 
-    ans: i32 = PyLong_AsLongLong(pValue)
+    ans: i64 = PyLong_AsLongLong(pValue)
     print("Ans is", ans)
-    assert ans == 5
+    assert ans == i64(5)
 
 
 def main0():

diff --git a/src/bin/lpython.cpp b/src/bin/lpython.cpp
@@ -321,6 +321,7 @@ int emit_c(const std::string &infile,
     pass_manager.use_default_passes(true);
     compiler_options.po.always_run = true;
     compiler_options.po.run_fun = "f";
+    compiler_options.po.c_skip_bindpy_pass = true;
 
     pass_manager.apply_passes(al, asr, compiler_options.po, diagnostics);
 
@@ -370,6 +371,7 @@ int emit_c_to_file(const std::string &infile, const std::string &outfile,
 
     compiler_options.po.run_fun = "f";
     compiler_options.po.always_run = true;
+    compiler_options.po.c_skip_bindpy_pass = true;
 
     pass_manager.use_default_passes(true);
     pass_manager.apply_passes(al, asr, compiler_options.po, diagnostics);
@@ -1130,7 +1132,7 @@ int compile_python_using_llvm(
         LCompilers::LPython::DynamicLibrary cpython_lib;
         LCompilers::LPython::DynamicLibrary symengine_lib;
 
-        if (compiler_options.enable_cpython) {
+        if (compiler_options.po.enable_cpython) {
             LCompilers::LPython::open_cpython_library(cpython_lib);
         }
         if (compiler_options.enable_symengine) {
@@ -1149,7 +1151,7 @@ int compile_python_using_llvm(
             e.execfn<void>("__module___main_____main__global_stmts");
         }
 
-        if (compiler_options.enable_cpython) {
+        if (compiler_options.po.enable_cpython) {
             LCompilers::LPython::close_cpython_library(cpython_lib);
         }
         if (compiler_options.enable_symengine) {
@@ -1533,7 +1535,7 @@ int link_executable(const std::vector<std::string> &infiles,
                 cmd += " -L$CONDA_PREFIX/lib -Wl,-rpath -Wl,$CONDA_PREFIX/lib -lsymengine";
             }
 
-            if (compiler_options.enable_cpython) {
+            if (compiler_options.po.enable_cpython) {
                 std::string py_version = "3.10";
                 std::string py_flags = R"(-I $CONDA_PREFIX/include/python)" + py_version + R"( -L$CONDA_PREFIX/lib -Wl,-rpath -Wl,$CONDA_PREFIX/lib -lpython)" + py_version + R"()";
                 if (compiler_options.link_numpy) {
@@ -1592,7 +1594,7 @@ int link_executable(const std::vector<std::string> &infiles,
         if (compiler_options.enable_symengine) {
             cmd += " -L$CONDA_PREFIX/lib -Wl,-rpath -Wl,$CONDA_PREFIX/lib -lsymengine";
         }
-        if (compiler_options.enable_cpython) {
+        if (compiler_options.po.enable_cpython) {
             std::string py_version = "3.10";
             std::string py_flags = R"(-I $CONDA_PREFIX/include/python)" + py_version + R"( -L$CONDA_PREFIX/lib -Wl,-rpath -Wl,$CONDA_PREFIX/lib -lpython)" + py_version + R"()";
             if (compiler_options.link_numpy) {
@@ -1919,7 +1921,7 @@ int main(int argc, char *argv[])
         app.add_flag("--dump-all-passes", compiler_options.po.dump_all_passes, "Apply all the passes and dump the ASR into a file");
         app.add_flag("--dump-all-passes-fortran", compiler_options.po.dump_fortran, "Apply all passes and dump the ASR after each pass into fortran file");
         app.add_flag("--cumulative", compiler_options.po.pass_cumulative, "Apply all the passes cumulatively till the given pass");
-        app.add_flag("--enable-cpython", compiler_options.enable_cpython, "Enable CPython runtime");
+        app.add_flag("--enable-cpython", compiler_options.po.enable_cpython, "Enable CPython runtime");
         app.add_flag("--enable-symengine", compiler_options.enable_symengine, "Enable Symengine runtime");
         app.add_flag("--link-numpy", compiler_options.link_numpy, "Enable NumPy runtime (implies --enable-cpython)");
         app.add_flag("--separate-compilation", separate_compilation, "Generates unique names for all the symbols");
@@ -1981,7 +1983,7 @@ int main(int argc, char *argv[])
         }
 
         if (compiler_options.link_numpy) {
-            compiler_options.enable_cpython = true;
+            compiler_options.po.enable_cpython = true;
         }
 
         if (arg_version) {

diff --git a/src/libasr/CMakeLists.txt b/src/libasr/CMakeLists.txt
@@ -37,6 +37,7 @@ set(SRC
     pass/for_all.cpp
     pass/while_else.cpp
     pass/global_stmts.cpp
+    pass/python_bind.cpp
     pass/select_case.cpp
     pass/init_expr.cpp
     pass/implied_do_loops.cpp

diff --git a/src/libasr/asr_utils.cpp b/src/libasr/asr_utils.cpp
@@ -1707,6 +1707,97 @@ void append_error(diag::Diagnostics& diag, const std::string& msg,
 //Initialize pointer to zero so that it can be initialized in first call to get_instance
 ASRUtils::LabelGenerator* ASRUtils::LabelGenerator::label_generator = nullptr;
 
+ASR::expr_t *type_enum_to_asr_expr(Allocator &al, enum TTYPE_T t, const Location &l, std::string n,
+    SymbolTable *current_scope, ASR::intentType intent) {
+    ASR::ttype_t *type = nullptr;
+
+    Str s;
+    s.from_str(al, n);
+
+    switch (t) {
+    case VOID:
+        return nullptr;
+    case I1:
+        type = ASRUtils::TYPE(ASR::make_Logical_t(al, l, 4));
+        break;
+    case I8:
+        type = ASRUtils::TYPE(ASR::make_Integer_t(al, l, 1));
+        break;
+    case I32:
+        type = ASRUtils::TYPE(ASR::make_Integer_t(al, l, 4));
+        break;
+    case I64:
+        type = ASRUtils::TYPE(ASR::make_Integer_t(al, l, 8));
+        break;
+    case U8:
+        type = ASRUtils::TYPE(ASR::make_UnsignedInteger_t(al, l, 1));
+        break;
+    case U32:
+        type = ASRUtils::TYPE(ASR::make_UnsignedInteger_t(al, l, 4));
+        break;
+    case U64:
+        type = ASRUtils::TYPE(ASR::make_UnsignedInteger_t(al, l, 8));
+        break;
+    case F32:
+        type = ASRUtils::TYPE(ASR::make_Real_t(al, l, 4));
+        break;
+    case F64:
+        type = ASRUtils::TYPE(ASR::make_Real_t(al, l, 8));
+        break;
+    case STR:
+        type = ASRUtils::TYPE(ASR::make_Character_t(al, l, 1, -2, nullptr));
+        break;
+    case PTR:
+        type = ASRUtils::TYPE(ASR::make_CPtr_t(al, l));
+        break;
+    case PTR_TO_PTR:
+        type = ASRUtils::TYPE(ASR::make_Pointer_t(al, l, ASRUtils::TYPE(ASR::make_CPtr_t(al, l))));
+        break;
+    }
+    LCOMPILERS_ASSERT(type);
+    ASR::symbol_t *v = ASR::down_cast<ASR::symbol_t>(ASR::make_Variable_t(al, l, current_scope, s.c_str(al), nullptr,
+                                                            0, intent, nullptr, nullptr, ASR::storage_typeType::Default,
+                                                            type, nullptr, ASR::abiType::BindC, ASR::Public,
+                                                            ASR::presenceType::Required, true));
+    current_scope->add_symbol(n, v);
+    return ASRUtils::EXPR(ASR::make_Var_t(al, l, v));
+}
+
+void declare_function(Allocator &al, ASRFunc fn, const Location &l, SymbolTable *parent_scope,
+    std::string header_name) {
+    Str s;
+    char *c_header = nullptr;
+    if (header_name != "") {
+        s.from_str(al, header_name);
+        c_header = s.c_str(al);
+    }
+    s.from_str(al, fn.m_name);
+    Vec<ASR::expr_t*> args;
+    args.reserve(al, fn.args.size());
+    SymbolTable *current_scope = al.make_new<SymbolTable>(parent_scope);
+    int c = 0;
+    for (auto j: fn.args) {
+        args.push_back(al, type_enum_to_asr_expr(al, j, l, fn.m_name + std::to_string(++c), current_scope,
+                                ASRUtils::intent_in));
+    }
+    ASR::expr_t *retval = type_enum_to_asr_expr(al, fn.retvar, l, "_lpython_return_variable", current_scope,
+                                ASRUtils::intent_return_var);
+    char *fn_name = s.c_str(al);
+    ASR::asr_t *f = ASRUtils::make_Function_t_util(al, l, current_scope, fn_name, nullptr, 0, args.p, args.n,
+                                nullptr, 0, retval, ASR::abiType::BindC, ASR::accessType::Public,
+                                ASR::deftypeType::Interface, nullptr, false, false, false, false, false, nullptr, 0,
+                                false, false, false, c_header);
+
+    parent_scope->add_symbol(fn.m_name, ASR::down_cast<ASR::symbol_t>(f));
+}
+
+void declare_functions(Allocator &al, std::vector<ASRFunc> fns, const Location &l, SymbolTable *parent_scope,
+    std::string header_name) {
+    for (auto i: fns) {
+        declare_function(al, i, l, parent_scope, header_name);
+    }
+}
+
 } // namespace ASRUtils
 
 

diff --git a/src/libasr/asr_utils.h b/src/libasr/asr_utils.h
@@ -5415,6 +5415,41 @@ static inline bool is_argument_of_type_CPtr(ASR::expr_t *var) {
     return is_argument;
 }
 
+enum TTYPE_T {
+    VOID,
+    I1,
+    I8,
+    STR,
+    I32,
+    I64,
+    U8,
+    U32,
+    U64,
+    F32,
+    F64,
+    PTR,
+    PTR_TO_PTR,
+};
+
+typedef struct {
+    std::string m_name;
+    std::vector<enum TTYPE_T> args;
+    enum TTYPE_T retvar;
+} ASRFunc;
+
+// Create a variable with name `n` and type `t` and add the symbol into `currect_scope`
+// Returns the variable
+ASR::expr_t *type_enum_to_asr_expr(Allocator &al, enum TTYPE_T t, const Location &l, std::string n,
+    SymbolTable *current_scope, ASR::intentType intent);
+
+// created a BindC Interface function decleration in the `parent_scope`
+void declare_function(Allocator &al, ASRFunc fn, const Location &l, SymbolTable *parent_scope,
+    std::string header_name="");
+
+// created a BindC Interface functions decleration in the `parent_scope`
+void declare_functions(Allocator &al, std::vector<ASRFunc> fns, const Location &l, SymbolTable *parent_scope,
+    std::string header_name="");
+
 } // namespace ASRUtils
 
 } // namespace LCompilers

diff --git a/src/libasr/codegen/asr_to_c.cpp b/src/libasr/codegen/asr_to_c.cpp
@@ -820,7 +820,7 @@ R"(
         }
 
         std::string body;
-        if (compiler_options.enable_cpython) {
+        if (compiler_options.po.enable_cpython) {
             headers.insert("Python.h");
             body += R"(
     Py_Initialize();
@@ -851,7 +851,7 @@ R"(    // Initialise Numpy
             body += src;
         }
 
-        if (compiler_options.enable_cpython) {
+        if (compiler_options.po.enable_cpython) {
             body += R"(
     if (Py_FinalizeEx() < 0) {
         fprintf(stderr,"BindPython: Unknown Error\n");

diff --git a/src/libasr/pass/pass_manager.h b/src/libasr/pass/pass_manager.h
@@ -54,6 +54,7 @@
 #include <libasr/pass/replace_print_struct_type.h>
 #include <libasr/pass/promote_allocatable_to_nonallocatable.h>
 #include <libasr/pass/replace_function_call_in_declaration.h>
+#include <libasr/pass/python_bind.h>
 #include <libasr/codegen/asr_to_fortran.h>
 #include <libasr/asr_verify.h>
 #include <libasr/pickle.h>
@@ -79,6 +80,7 @@ namespace LCompilers {
             {"do_loops", &pass_replace_do_loops},
             {"while_else", &pass_while_else},
             {"global_stmts", &pass_wrap_global_stmts},
+            {"python_bind", &pass_python_bind},
             {"implied_do_loops", &pass_replace_implied_do_loops},
             {"array_op", &pass_replace_array_op},
             {"symbolic", &pass_replace_symbolic},
@@ -206,6 +208,7 @@ namespace LCompilers {
         PassManager(): apply_default_passes{false},
             c_skip_pass{false} {
             _passes = {
+                "python_bind",
                 "nested_vars",
                 "global_stmts",
                 "transform_optional_argument_functions",