pydantic_model_creator does not support Forward references using ReverseRelation in different files

[eyllanesc-JE]

Describe the bug

If I use Forward references in the same file as I use ReverseRelation this does not generate problems.

import json

from tortoise.contrib.pydantic import pydantic_model_creator
from tortoise import Tortoise
from tortoise import Model
from tortoise import fields


class Foo(Model):
    id = fields.IntField(primary_key=True)
    name = fields.CharField(max_length=255)
    bar = fields.ReverseRelation["Bar"]


class Bar(Model):
    id = fields.IntField(primary_key=True)
    name = fields.CharField(max_length=255)
    foo: fields.ForeignKeyRelation[Foo] = fields.ForeignKeyField(
        "models.Foo", on_delete=fields.CASCADE
    )


Tortoise.init_models(models_paths=["__main__"], app_label="models")
BarIn = pydantic_model_creator(Bar, name="BarIn")
print(json.dumps(BarIn.model_json_schema(), indent=4))

Output:

{
    "$defs": {
        "Foo_tpibnp_leaf": {
            "additionalProperties": false,
            "properties": {
                "id": {
                    "maximum": 2147483647,
                    "minimum": -2147483648,
                    "title": "Id",
                    "type": "integer"
                },
                "name": {
                    "maxLength": 255,
                    "title": "Name",
                    "type": "string"
                }
            },
            "required": [
                "id",
                "name"
            ],
            "title": "Foo",
            "type": "object"
        }
    },
    "additionalProperties": false,
    "properties": {
        "id": {
            "maximum": 2147483647,
            "minimum": -2147483648,
            "title": "Id",
            "type": "integer"
        },
        "name": {
            "maxLength": 255,
            "title": "Name",
            "type": "string"
        },
        "foo": {
            "$ref": "#/$defs/Foo_tpibnp_leaf"
        }
    },
    "required": [
        "id",
        "name",
        "foo"
    ],
    "title": "BarIn",
    "type": "object"
}

But if I separate them into different files:

├── models
│      ├───__init__.py
│      ├───bar.py
│      └───foo.py
└── main.py

foo.py

from tortoise import Model
from tortoise import fields

from models.bar import Bar


class Foo(Model):
    id = fields.IntField(primary_key=True)
    name = fields.CharField(max_length=255)
    bar = fields.ReverseRelation[Bar]

bar.py

from __future__ import annotations

from typing import TYPE_CHECKING

from tortoise import Model
from tortoise import fields

if TYPE_CHECKING:
    from models.foo import Foo


class Bar(Model):
    id = fields.IntField(primary_key=True)
    name = fields.CharField(max_length=255)
    foo: fields.ForeignKeyRelation[Foo] = fields.ForeignKeyField(
        "models.Foo", on_delete=fields.CASCADE
    )

main.py

from __future__ import annotations
import json

from tortoise.contrib.pydantic import pydantic_model_creator
from tortoise import Tortoise
from models import foo, bar
from models.bar import Bar

Tortoise.init_models(models_paths=[foo, bar], app_label="models")
BarIn = pydantic_model_creator(Bar, name="BarIn")
print(json.dumps(BarIn.model_json_schema(), indent=4))

Getting the following:

Traceback (most recent call last):
  File "c:\Users\HP\Documents\apps\test_tortoirse_orm\main.py", line 10, in <module>
    BarIn = pydantic_model_creator(Bar, name="BarIn")
            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\HP\Documents\apps\test_tortoirse_orm\.venv\Lib\site-packages\tortoise\contrib\pydantic\creator.py", line 625, in pydantic_model_creator
    pmc = PydanticModelCreator(
          ^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\HP\Documents\apps\test_tortoirse_orm\.venv\Lib\site-packages\tortoise\contrib\pydantic\creator.py", line 300, in __init__
    self._annotations = get_annotations(cls)
                        ^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\HP\Documents\apps\test_tortoirse_orm\.venv\Lib\site-packages\tortoise\contrib\pydantic\utils.py", line 15, in get_annotations
    return typing.get_type_hints(method or cls)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\HP\AppData\Local\Programs\Python\Python312\Lib\typing.py", line 2273, in get_type_hints
    value = _eval_type(value, base_globals, base_locals, base.__type_params__)
            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\HP\AppData\Local\Programs\Python\Python312\Lib\typing.py", line 415, in _eval_type
    return t._evaluate(globalns, localns, type_params, recursive_guard=recursive_guard)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\HP\AppData\Local\Programs\Python\Python312\Lib\typing.py", line 947, in _evaluate
    eval(self.__forward_code__, globalns, localns),
    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "<string>", line 1, in <module>
NameError: name 'Foo' is not defined

[eyllanesc-JE]

Checking I found that the problem has its root in the following code:

def get_annotations(cls: "Type[Model]", method: Optional[Callable] = None) -> Dict[str, Any]:
    """
    Get all annotations including base classes
    :param cls: The model class we need annotations from
    :param method: If specified, we try to get the annotations for the callable
    :return: The list of annotations
    """
    return typing.get_type_hints(method or cls)

And looking at the change history I implemented the following modification:

def get_annotations(cls: "Type[Model]", method: Optional[Callable] = None) -> Dict[str, Any]:
    """
    Get all annotations including base classes
    :param cls: The model class we need annotations from
    :param method: If specified, we try to get the annotations for the callable
    :return: The list of annotations
    """
    localns = (
        tortoise.Tortoise.apps.get(cls._meta.app, None)
        if cls._meta.app
        else None
    )
    return typing.get_type_hints(method or cls, localns=localns)

Solving my problem and getting:

{
    "$defs": {
        "Foo_ijnnvp_leaf": {
            "additionalProperties": false,
            "properties": {
                "id": {
                    "maximum": 2147483647,
                    "minimum": -2147483648,
                    "title": "Id",
                    "type": "integer"
                },
                "name": {
                    "maxLength": 255,
                    "title": "Name",
                    "type": "string"
                }
            },
            "required": [
                "id",
                "name"
            ],
            "title": "Foo",
            "type": "object"
        }
    },
    "additionalProperties": false,
    "properties": {
        "id": {
            "maximum": 2147483647,
            "minimum": -2147483648,
            "title": "Id",
            "type": "integer"
        },
        "name": {
            "maxLength": 255,
            "title": "Name",
            "type": "string"
        },
        "foo": {
            "$ref": "#/$defs/Foo_ijnnvp_leaf"
        }
    },
    "required": [
        "id",
        "name",
        "foo"
    ],
    "title": "BarIn",
    "type": "object"
}

My solution is different from what was generated by issue #1552, It indicates there that the error is due to setting the globalns.

[henadzit]

@eyllanesc-JE is it the complete traceback?

Traceback (most recent call last):
....
  File "<string>", line 1, in <module>
NameError: name 'Foo' is not defined

[markus-96]

just change

if TYPE_CHECKING:
    from foo import Foo

to

from foo import Foo

during runtime, TYPE_CHECKING is always False. So it won't import Foo.

Full example, working with python3.12:

first option with ForeignKeyRelation

main.py

import json

from typing import Callable, Coroutine

from tortoise import run_async, Tortoise
from tortoise.contrib.pydantic import pydantic_model_creator

from bar import Bar


def run(func: Callable[..., Coroutine]) -> None:
    run_async(func())

async def do_stuff():
    await Tortoise.init(
        db_url='sqlite:///:memory:',
        modules={'models': ['foo', 'bar']}
    )
    BarIn = pydantic_model_creator(Bar, name="BarIn")
    print(json.dumps(BarIn.model_json_schema(), indent=2))


if __name__ == '__main__':
    run(do_stuff)

foo.py

from tortoise import fields
from tortoise import Model


class Foo(Model):
    id = fields.IntField(primary_key=True)
    name = fields.CharField(max_length=255)

bar.py

from tortoise import fields
from tortoise import Model

from foo import Foo


class Bar(Model):
    id = fields.IntField(primary_key=True)
    name = fields.CharField(max_length=255)
    foo: fields.ForeignKeyRelation[Foo] = fields.ForeignKeyField(
        "models.Foo", on_delete=fields.CASCADE
    )

second option without ForeignKeyRelation

bar.py

from tortoise import fields
from tortoise import Model


class Bar(Model):
    id = fields.IntField(primary_key=True)
    name = fields.CharField(max_length=255)
    foo = fields.ForeignKeyField(
        "models.Foo", on_delete=fields.CASCADE
    )

[eyllanesc-JE]

@markus-96 Thanks for the comments. I realized that I have not focused my problem correctly. It is not so much the ForeignKeyField but the ForwardRef that is not handled correctly by pydantic_model_creator.

• I use TYPE_CHECKING for my projects to avoid circular imports.
• I am using mypy so I need the typehints so that is why I cannot use the second option.

I will improve my post to focus only on ForwardRef.

[henadzit]

@eyllanesc-JE if you move from models.foo import Foo WITHOUT the if TYPE_CHECKING: after the Bar definition:

class Bar(Model):
    foo: fields.ForeignKeyRelation["Foo"] = fields.ForeignKeyField(
        "models.Foo", on_delete=fields.CASCADE
    )


from models.foo import Foo

Does it fix the issue?

[eyllanesc-JE]

@henadzit No, I get the same error.

[markus-96]

the following is working, but a little bit ugly, but it should be more compliant to PEP8 (imports to the top of the file):

from typing import TYPE_CHECKING

import tortoise
from tortoise import fields
from tortoise import Model
from typing_extensions import TypeVar

if TYPE_CHECKING:
    from models.foo import Foo
else:
    Foo = TypeVar('Foo', bound=Model)


class Bar(Model):
    id = fields.IntField(primary_key=True)
    name = fields.CharField(max_length=255)
    foo: fields.ForeignKeyRelation["Foo"] = fields.ForeignKeyField(
        "models.Foo", on_delete=fields.CASCADE
    )

The problem I have with the proposed solution of @eyllanesc-JE is that imports like import bar and then using them like ForeignKeyRelation["bar.Bar"] will not work. So it would need to be clearly documented how to import reliant models.

Also, if you have a structure with multiple apps, you will not be able to reference any models from the other app. ie:

models
|- bar.py
|- foo.py
models2
|- foo2.py
main.py

(bar.py and foo.py are the same as in the comments above...)

foo2.py

[...]
class Foo2(Model):
    id = fields.IntField(primary_key=True)
    name_s = fields.CharField(max_length=255)
    bar: fields.ForeignKeyRelation["Bar"] = fields.ForeignKeyField(
        "models.Bar", on_delete=fields.CASCADE, related_name='foos2'
    )

main.py

import json
from tortoise.contrib.pydantic import pydantic_model_creator
from tortoise import Tortoise
from models.bar import Bar


app_modules = {'models': ['models.foo', 'models.bar'], 'models2': ['models2.foo2']}
for name, modules in app_modules.items():
    Tortoise.init_models(modules, name, _init_relations=False)
for name, modules in app_modules.items():
    Tortoise.init_models(modules, name)
BarIn = pydantic_model_creator(Bar, name="BarIn")
print(json.dumps(BarIn.model_json_schema(), indent=4))

So, I think the ugly way of "importing" stuff is the most reliable one.

Full example

models/bar.py

from typing import TYPE_CHECKING

import tortoise
from tortoise import fields
from tortoise import Model


if TYPE_CHECKING:
    from models.foo import Foo
    from models2.foo2 import Foo2
else:
    Foo = TypeVar('Foo', bound=Model)
    Foo2 = TypeVar('Foo2', bound=Model)


class Bar(Model):
    id = fields.IntField(primary_key=True)
    name = fields.CharField(max_length=255)
    foo: fields.ForeignKeyRelation["Foo"] = fields.ForeignKeyField(
        "models.Foo", on_delete=fields.CASCADE
    )
    foo2: fields.ForeignKeyRelation["Foo2"] = fields.ForeignKeyField(
        "models2.Foo2", on_delete=fields.CASCADE
    )

models/foo.py

from typing import TYPE_CHECKING

import tortoise
from tortoise import fields
from tortoise import Model

if TYPE_CHECKING:
    from models.bar import Bar
else:
    Bar = TypeVar('Bar', bound=Model)


class Foo(Model):
    id = fields.IntField(primary_key=True)
    name = fields.CharField(max_length=255)
    bar: fields.ForeignKeyRelation["Bar"] = fields.ForeignKeyField(
        "models.Bar", on_delete=fields.CASCADE
    )

models2/foo2.py

from typing import TYPE_CHECKING

import tortoise
from tortoise import fields
from tortoise import Model

if TYPE_CHECKING:
    from models.bar import Bar
else:
    Bar = TypeVar('Bar', bound=Model)


class Foo2(Model):
    id = fields.IntField(primary_key=True)
    name_s = fields.CharField(max_length=255)
    bar: fields.ForeignKeyRelation["Bar"] = fields.ForeignKeyField(
        "models.Bar", on_delete=fields.CASCADE, related_name='foos2'
    )

main.py

import json
from tortoise.contrib.pydantic import pydantic_model_creator
from tortoise import Tortoise
from models.bar import Bar


app_modules = {'models': ['models.foo', 'models.bar'], 'models2': ['models2.foo2']}
for name, modules in app_modules.items():
    Tortoise.init_models(modules, name, _init_relations=False)
for name, modules in app_modules.items():
    Tortoise.init_models(modules, name)
BarIn = pydantic_model_creator(Bar, name="BarIn")
print(json.dumps(BarIn.model_json_schema(), indent=4))

(Edit: simpler main.py)
(Edit2: much simpler import-section)