diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index cc021ac2..9e080b2d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -12,7 +12,7 @@ on: jobs: julia: - name: Julia (${{ matrix.jlversion }}, ${{ matrix.os }}, ${{ matrix.pythonexe }}) + name: Julia (${{ matrix.jlversion }}, ${{ matrix.os }}, ${{ matrix.pythonexe }} ${{ matrix.pyversion}}) runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -21,11 +21,18 @@ jobs: os: [ubuntu-latest, windows-latest, macos-latest] jlversion: ['1','1.10'] pythonexe: ['@CondaPkg'] + pyversion: ['3'] include: - arch: x64 os: ubuntu-latest jlversion: '1' pythonexe: python + pyversion: '3' + - arch: x64 + os: ubuntu-latest + jlversion: '1' + pythonexe: python + pyversion: '3.14t' steps: - uses: actions/checkout@v6 @@ -43,20 +50,27 @@ jobs: env: PYTHON: python + - name: Install Python + id: setup-python + uses: actions/setup-python@v6 + if: ${{ matrix.pythonexe == 'python' }} + with: + python-version: ${{ matrix.pyversion }} + - name: Build PyCall if: ${{ matrix.pythonexe == 'python' }} run: | julia --project=test -e 'import Pkg; Pkg.build("PyCall")' env: - PYTHON: python + PYTHON: ${{ steps.setup-python.outputs.python-path }} - name: Run tests uses: julia-actions/julia-runtest@v1 env: JULIA_DEBUG: PythonCall JULIA_NUM_THREADS: '2' - PYTHON: python - JULIA_PYTHONCALL_EXE: ${{ matrix.pythonexe }} + PYTHON: ${{ steps.setup-python.outputs.python-path }} + JULIA_PYTHONCALL_EXE: ${{ case(matrix.pythonexe == 'python', steps.setup-python.outputs.python-path, matrix.pythonexe) }} - name: Process coverage uses: julia-actions/julia-processcoverage@v1 @@ -79,6 +93,9 @@ jobs: - os: ubuntu-latest pyversion: '3' juliaexe: julia + - os: ubuntu-latest + pyversion: '3.14t' + juliaexe: '@JuliaPkg' env: MANUAL_TEST_PROJECT: /tmp/juliacall-test-project PYTHON_JULIACALL_THREADS: '2' diff --git a/CHANGELOG.md b/CHANGELOG.md index b8e6aa0e..459e14f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ to the corresponding NumPy dtype, like `numpy.dtype(jl.Int)`. * JuliaCall now launches Julia with 1 thread by default. * Added options `trace_compile` and `trace_compile_timing` to JuliaCall. +* Initial experimental support for free-threaded Python 3.14. * Bug fixes. ## 0.9.31 (2025-12-17) diff --git a/src/C/consts.jl b/src/C/consts.jl index 4560e928..7d174968 100644 --- a/src/C/consts.jl +++ b/src/C/consts.jl @@ -124,6 +124,20 @@ end type::Ptr{Cvoid} = C_NULL # really is Ptr{PyObject} or Ptr{PyTypeObject} but Julia 1.3 and below get the layout incorrect when circular types are involved end +@kwdef struct PyMutex + bits::Cuchar = 0 +end + +@kwdef struct PyObjectFT + tid::Csize_t = 0 + flags::Cushort = 0 + mutex::PyMutex = PyMutex() + gc_bits::Cuchar = 0 + ref_local::Cuint = 0 + ref_shared::Py_ssize_t = 0 + type::Ptr{Cvoid} = C_NULL # really is Ptr{PyObject} or Ptr{PyTypeObject} but Julia 1.3 and below get the layout incorrect when circular types are involved +end + const PyPtr = Ptr{PyObject} const PyNULL = PyPtr(0) @@ -139,6 +153,11 @@ Base.unsafe_convert(::Type{PyPtr}, o::PyObjectRef) = o.ptr size::Py_ssize_t = 0 end +@kwdef struct PyVarObjectFT + ob_base::PyObjectFT = PyObjectFT() + size::Py_ssize_t = 0 +end + @kwdef struct PyMethodDef name::Cstring = C_NULL meth::Ptr{Cvoid} = C_NULL @@ -249,6 +268,16 @@ end weakreflist::PyPtr = PyNULL end +@kwdef struct PyMemoryViewObjectFT + ob_base::PyVarObjectFT = PyVarObjectFT() + mbuf::PyPtr = PyNULL + hash::Py_hash_t = 0 + flags::Cint = 0 + exports::Py_ssize_t = 0 + view::Py_buffer = Py_buffer() + weakreflist::PyPtr = PyNULL +end + @kwdef struct PyTypeObject ob_base::PyVarObject = PyVarObject() name::Cstring = C_NULL @@ -327,6 +356,11 @@ end value::T end +@kwdef struct PySimpleObjectFT{T} + ob_base::PyObjectFT = PyObjectFT() + value::T +end + @kwdef struct PyArrayInterface two::Cint = 0 nd::Cint = 0 diff --git a/src/C/context.jl b/src/C/context.jl index 9d1f5d30..4d83cf2a 100644 --- a/src/C/context.jl +++ b/src/C/context.jl @@ -17,6 +17,7 @@ A handle to a loaded instance of libpython, its interpreter, function pointers, pyhome_w::Any = missing which::Symbol = :unknown # :CondaPkg, :PyCall, :embedded or :unknown version::Union{VersionNumber,Missing} = missing + is_free_threaded::Bool = false end const CTX = Context() @@ -312,10 +313,11 @@ function init_context() v"3.10" ≤ CTX.version < v"4" || error( "Only Python 3.10+ is supported, this is Python $(CTX.version) at $(CTX.exe_path===missing ? "unknown location" : CTX.exe_path).", ) + CTX.is_free_threaded = occursin("free-threading build", verstr) launch_on_main_thread(Threads.threadid()) # makes on_main_thread usable - @debug "Initialized PythonCall.jl" CTX.is_embedded CTX.is_initialized CTX.exe_path CTX.lib_path CTX.lib_ptr CTX.pyprogname CTX.pyhome CTX.version + @debug "Initialized PythonCall.jl" CTX.is_embedded CTX.is_initialized CTX.exe_path CTX.lib_path CTX.lib_ptr CTX.pyprogname CTX.pyhome CTX.version CTX.is_free_threaded return end diff --git a/src/C/extras.jl b/src/C/extras.jl index cf46b6b8..d74d2d43 100644 --- a/src/C/extras.jl +++ b/src/C/extras.jl @@ -1,6 +1,46 @@ asptr(x) = Base.unsafe_convert(PyPtr, x) -Py_Type(x) = Base.GC.@preserve x PyPtr(UnsafePtr(asptr(x)).type[!]) +# Free-threaded CPython builds ("3.14t") currently have different C struct layouts, +# but there is no stable ABI yet. To keep the code manageable, we centralize the +# branching in a single macro that rewrites type names in the expression. +const _FT_TYPE_REPLACEMENTS = Dict{Symbol,Symbol}( + :PyObject => :PyObjectFT, + :PyVarObject => :PyVarObjectFT, + :PyMemoryViewObject => :PyMemoryViewObjectFT, + :PySimpleObject => :PySimpleObjectFT, + # Used from JlWrap/C.jl via `C.@ft`. + :PyJuliaValueObject => :PyJuliaValueObjectFT, +) + +function _ft_transform(ex) + if ex isa Symbol + return get(_FT_TYPE_REPLACEMENTS, ex, ex) + elseif ex isa QuoteNode + return QuoteNode(_ft_transform(ex.value)) + elseif ex isa Expr + return Expr(ex.head, map(_ft_transform, ex.args)...) + else + return ex + end +end + +""" + @ft expr + +Evaluate `expr`, but when `CTX.is_free_threaded` is true (CPython "free-threaded" +builds), rewrite internal type names like `PyObject` → `PyObjectFT` inside the +expression. + +This keeps free-threaded branching centralized, so we don't scatter `if +CTX.is_free_threaded` throughout the code. +""" +macro ft(ex) + ex_ft = _ft_transform(ex) + m = @__MODULE__ + return esc(:($m.CTX.is_free_threaded ? $ex_ft : $ex)) +end + +Py_Type(x) = Base.GC.@preserve x @ft PyPtr(UnsafePtr{PyObject}(asptr(x)).type[!]) PyObject_Type(x) = Base.GC.@preserve x (t = Py_Type(asptr(x)); Py_IncRef(t); t) @@ -8,37 +48,41 @@ Py_TypeCheck(o, t) = Base.GC.@preserve o t PyType_IsSubtype(Py_Type(asptr(o)), a Py_TypeCheckFast(o, f::Integer) = Base.GC.@preserve o PyType_IsSubtypeFast(Py_Type(asptr(o)), f) PyType_IsSubtypeFast(t, f::Integer) = - Base.GC.@preserve t Cint(!iszero(UnsafePtr{PyTypeObject}(asptr(t)).flags[] & f)) + Base.GC.@preserve t Cint(!iszero(PyType_GetFlags(asptr(t)) & f)) -PyMemoryView_GET_BUFFER(m) = Base.GC.@preserve m Ptr{Py_buffer}(UnsafePtr{PyMemoryViewObject}(asptr(m)).view) +PyMemoryView_GET_BUFFER(m) = Base.GC.@preserve m @ft Ptr{Py_buffer}(UnsafePtr{PyMemoryViewObject}(asptr(m)).view) PyType_CheckBuffer(t) = Base.GC.@preserve t begin - p = UnsafePtr{PyTypeObject}(asptr(t)).as_buffer[] - return p != C_NULL && p.get[!] != C_NULL + getbuf = PyType_GetSlot(asptr(t), Py_bf_getbuffer) + return getbuf != C_NULL end PyObject_CheckBuffer(o) = Base.GC.@preserve o PyType_CheckBuffer(Py_Type(asptr(o))) PyObject_GetBuffer(_o, b, flags) = Base.GC.@preserve _o begin o = asptr(_o) - p = UnsafePtr{PyTypeObject}(Py_Type(o)).as_buffer[] - if p == C_NULL || p.get[!] == C_NULL - PyErr_SetString( - POINTERS.PyExc_TypeError, - "a bytes-like object is required, not '$(String(UnsafePtr{PyTypeObject}(Py_Type(o)).name[]))'", - ) + getbuf = PyType_GetSlot(Py_Type(o), Py_bf_getbuffer) + if getbuf == C_NULL + # TODO: we can drop this branch and just use PyType_GetName once we stop + # supporting python 3.10 + msg = if CTX.is_free_threaded + "a bytes-like object is required" + else + "a bytes-like object is required, not '$(String(UnsafePtr{PyTypeObject}(Py_Type(o)).name[]))'" + end + PyErr_SetString(POINTERS.PyExc_TypeError, msg) return Cint(-1) end - return ccall(p.get[!], Cint, (PyPtr, Ptr{Py_buffer}, Cint), o, b, flags) + return ccall(getbuf, Cint, (PyPtr, Ptr{Py_buffer}, Cint), o, b, flags) end PyBuffer_Release(_b) = begin b = UnsafePtr(Base.unsafe_convert(Ptr{Py_buffer}, _b)) o = b.obj[] o == C_NULL && return - p = UnsafePtr{PyTypeObject}(Py_Type(o)).as_buffer[] - if (p != C_NULL && p.release[!] != C_NULL) - ccall(p.release[!], Cvoid, (PyPtr, Ptr{Py_buffer}), o, b) + releasebuf = PyType_GetSlot(Py_Type(o), Py_bf_releasebuffer) + if releasebuf != C_NULL + ccall(releasebuf, Cvoid, (PyPtr, Ptr{Py_buffer}), o, b) end b.obj[] = C_NULL Py_DecRef(o) @@ -65,7 +109,7 @@ function PyOS_RunInputHook() end function PySimpleObject_GetValue(::Type{T}, o) where {T} - Base.GC.@preserve o UnsafePtr{PySimpleObject{T}}(asptr(o)).value[!] + Base.GC.@preserve o @ft UnsafePtr{PySimpleObject{T}}(asptr(o)).value[!] end # FAST REFCOUNTING diff --git a/src/C/pointers.jl b/src/C/pointers.jl index d7441960..9644329f 100644 --- a/src/C/pointers.jl +++ b/src/C/pointers.jl @@ -77,6 +77,8 @@ const CAPI_FUNC_SIGS = Dict{Symbol,Pair{Tuple,Type}}( :PyType_Ready => (PyPtr,) => Cint, :PyType_GenericNew => (PyPtr, PyPtr, PyPtr) => PyPtr, :PyType_FromSpec => (Ptr{Cvoid},) => PyPtr, + :PyType_GetFlags => (PyPtr,) => Culong, + :PyType_GetSlot => (PyPtr, Cint) => Ptr{Cvoid}, # MAPPING :PyMapping_HasKeyString => (PyPtr, Ptr{Cchar}) => Cint, :PyMapping_SetItemString => (PyPtr, Ptr{Cchar}, PyPtr) => Cint, diff --git a/src/JlWrap/C.jl b/src/JlWrap/C.jl index 50c17987..19ff2904 100644 --- a/src/JlWrap/C.jl +++ b/src/JlWrap/C.jl @@ -12,6 +12,12 @@ using Serialization: serialize, deserialize weaklist::C.PyPtr = C_NULL end +@kwdef struct PyJuliaValueObjectFT + ob_base::C.PyObjectFT = C.PyObjectFT() + value::Int = 0 + weaklist::C.PyPtr = C_NULL +end + const PyJuliaBase_Type = Ref(C.PyNULL) # we store the actual julia values here @@ -21,21 +27,24 @@ const PYJLVALUES = [] const PYJLFREEVALUES = Int[] function _pyjl_new(t::C.PyPtr, ::C.PyPtr, ::C.PyPtr) - o = ccall(UnsafePtr{C.PyTypeObject}(t).alloc[!], C.PyPtr, (C.PyPtr, C.Py_ssize_t), t, 0) + alloc = C.PyType_GetSlot(t, C.Py_tp_alloc) + alloc == C_NULL && return C.PyNULL + o = ccall(alloc, C.PyPtr, (C.PyPtr, C.Py_ssize_t), t, 0) o == C.PyNULL && return C.PyNULL - UnsafePtr{PyJuliaValueObject}(o).weaklist[] = C.PyNULL - UnsafePtr{PyJuliaValueObject}(o).value[] = 0 + C.@ft UnsafePtr{PyJuliaValueObject}(o).weaklist[] = C.PyNULL + C.@ft UnsafePtr{PyJuliaValueObject}(o).value[] = 0 return o end function _pyjl_dealloc(o::C.PyPtr) - idx = UnsafePtr{PyJuliaValueObject}(o).value[] + idx = C.@ft UnsafePtr{PyJuliaValueObject}(o).value[] if idx != 0 PYJLVALUES[idx] = nothing push!(PYJLFREEVALUES, idx) end - UnsafePtr{PyJuliaValueObject}(o).weaklist[!] == C.PyNULL || C.PyObject_ClearWeakRefs(o) - ccall(UnsafePtr{C.PyTypeObject}(C.Py_Type(o)).free[!], Cvoid, (C.PyPtr,), o) + (C.@ft UnsafePtr{PyJuliaValueObject}(o).weaklist[!]) == C.PyNULL || C.PyObject_ClearWeakRefs(o) + freeptr = C.PyType_GetSlot(C.Py_Type(o), C.Py_tp_free) + freeptr == C_NULL || ccall(freeptr, Cvoid, (C.PyPtr,), o) nothing end @@ -319,7 +328,7 @@ function init_c() C.PyMemberDef( name = pointer(_pyjlbase_weaklistoffset_name), typ = C.Py_T_PYSSIZET, - offset = fieldoffset(PyJuliaValueObject, 3), + offset = (C.@ft fieldoffset(PyJuliaValueObject, 3)), flags = C.Py_READONLY, ), C.PyMemberDef(), # NULL terminator @@ -341,7 +350,7 @@ function init_c() # Create PyType_Spec _pyjlbase_spec[] = C.PyType_Spec( name = pointer(_pyjlbase_name), - basicsize = sizeof(PyJuliaValueObject), + basicsize = (C.@ft sizeof(PyJuliaValueObject)), flags = C.Py_TPFLAGS_BASETYPE | C.Py_TPFLAGS_HAVE_VERSION_TAG, slots = pointer(_pyjlbase_slots), ) @@ -358,13 +367,13 @@ function __init__() init_c() end -PyJuliaValue_IsNull(o) = Base.GC.@preserve o UnsafePtr{PyJuliaValueObject}(C.asptr(o)).value[] == 0 +PyJuliaValue_IsNull(o) = Base.GC.@preserve o (C.@ft UnsafePtr{PyJuliaValueObject}(C.asptr(o)).value[]) == 0 -PyJuliaValue_GetValue(o) = Base.GC.@preserve o PYJLVALUES[UnsafePtr{PyJuliaValueObject}(C.asptr(o)).value[]] +PyJuliaValue_GetValue(o) = Base.GC.@preserve o PYJLVALUES[(C.@ft UnsafePtr{PyJuliaValueObject}(C.asptr(o)).value[])] PyJuliaValue_SetValue(_o, @nospecialize(v)) = Base.GC.@preserve _o begin o = C.asptr(_o) - idx = UnsafePtr{PyJuliaValueObject}(o).value[] + idx = C.@ft UnsafePtr{PyJuliaValueObject}(o).value[] if idx == 0 if isempty(PYJLFREEVALUES) push!(PYJLVALUES, v) @@ -373,7 +382,7 @@ PyJuliaValue_SetValue(_o, @nospecialize(v)) = Base.GC.@preserve _o begin idx = pop!(PYJLFREEVALUES) PYJLVALUES[idx] = v end - UnsafePtr{PyJuliaValueObject}(o).value[] = idx + C.@ft UnsafePtr{PyJuliaValueObject}(o).value[] = idx else PYJLVALUES[idx] = v end diff --git a/test/Compat.jl b/test/Compat.jl index 4ad57a5c..3f86f2d5 100644 --- a/test/Compat.jl +++ b/test/Compat.jl @@ -50,10 +50,11 @@ end end @testitem "PyCall.jl" begin - if (get(ENV, "CI", "") != "") && (ENV["JULIA_PYTHONCALL_EXE"] == "python") + if (get(ENV, "CI", "") != "") && (ENV["JULIA_PYTHONCALL_EXE"] == "python") && !PythonCall.C.CTX.is_free_threaded # Only run this test when we can guarantee PyCall and PythonCall are using the # same Python. Currently this only runs in CI, and if PythonCall is using the - # system Python installation. + # system Python installation. Also PyCall is not compatible with free-threaded + # python so we skip this too. using PyCall # Check they are indeed using the same Python. @test Base.get_extension(PythonCall, :PyCallExt).SAME[]