mayflowergmbh/bert-german-ler-onnx-int4

German Legal NER - ONNX INT4 Quantized

4-bit quantized ONNX version of elenanereiss/bert-german-ler for Named Entity Recognition in German legal texts.

Model Details

Performance

Metrics from the source model evaluated on the German LER test set:

Per-entity F1 (test set)

Entity Types (19 classes)

Usage

Requirements

pip install onnxruntime transformers numpy

Inference

import onnxruntime as ort
from transformers import AutoTokenizer
import numpy as np
import json

# Load tokenizer and model
tokenizer = AutoTokenizer.from_pretrained("mayflowergmbh/bert-german-ler-onnx-int4")
session = ort.InferenceSession("model_int4.onnx", providers=["CPUExecutionProvider"])

# Load label mapping
with open("config.json") as f:
    config = json.load(f)
id2label = config["id2label"]

# Tokenize input
text = "Herr Müller verstieß gegen § 36 Abs. 7 IfSG und wurde vom Bundesgerichtshof verurteilt."
inputs = tokenizer(text, return_tensors="np", padding=True, truncation=True)

# Run inference
outputs = session.run(None, {
    "input_ids": inputs["input_ids"].astype(np.int64),
    "attention_mask": inputs["attention_mask"].astype(np.int64),
    "token_type_ids": inputs["token_type_ids"].astype(np.int64),
})

# Decode predictions
predictions = np.argmax(outputs[0], axis=-1)[0]
tokens = tokenizer.convert_ids_to_tokens(inputs["input_ids"][0])

for token, pred_id in zip(tokens, predictions):
    label = id2label[str(pred_id)]
    if token not in ("[PAD]", "[CLS]", "[SEP]") and label != "O":
        print(f"{token:20s} {label}")

Output:

Müller               B-PER
§                    B-GS
36                   I-GS
Abs                  I-GS
.                    I-GS
7                    I-GS
I                    I-GS
##f                  I-GS
##SG                 I-GS
Bundes               B-GRT
##gerichtshof        I-GRT

Entity Extraction Helper

def extract_entities(text, tokenizer, session, id2label):
    inputs = tokenizer(text, return_tensors="np", padding=True, truncation=True)
    outputs = session.run(None, {
        "input_ids": inputs["input_ids"].astype(np.int64),
        "attention_mask": inputs["attention_mask"].astype(np.int64),
        "token_type_ids": inputs["token_type_ids"].astype(np.int64),
    })
    predictions = np.argmax(outputs[0], axis=-1)[0]
    tokens = tokenizer.convert_ids_to_tokens(inputs["input_ids"][0])

    entities = []
    current_entity = None
    current_tokens = []

    for token, pred_id in zip(tokens, predictions):
        if token in ("[PAD]", "[CLS]", "[SEP]"):
            continue
        label = id2label[str(pred_id)]

        if label.startswith("B-"):
            if current_entity:
                entities.append({
                    "entity": current_entity,
                    "text": tokenizer.convert_tokens_to_string(current_tokens).strip()
                })
            current_entity = label[2:]
            current_tokens = [token]
        elif label.startswith("I-") and current_entity == label[2:]:
            current_tokens.append(token)
        else:
            if current_entity:
                entities.append({
                    "entity": current_entity,
                    "text": tokenizer.convert_tokens_to_string(current_tokens).strip()
                })
                current_entity = None
                current_tokens = []

    if current_entity:
        entities.append({
            "entity": current_entity,
            "text": tokenizer.convert_tokens_to_string(current_tokens).strip()
        })

    return entities

entities = extract_entities(
    "Das Urteil des BGH vom 12.03.2021 (Az. III ZR 5/20) stützt sich auf § 280 Abs. 1 BGB.",
    tokenizer, session, id2label
)
for e in entities:
    print(f"[{e['entity']:>3s}] {e['text']}")

Output:

[GRT] BGH
[ RS] Az. III ZR 5 / 20
[ GS] § 280 Abs. 1 BGB

Batch Inference

texts = [
    "Der Kläger berief sich auf Art. 6 EMRK.",
    "Die Richterin Dr. Schmidt verwies auf das BVerfG-Urteil.",
]

inputs = tokenizer(texts, return_tensors="np", padding=True, truncation=True)
outputs = session.run(None, {
    "input_ids": inputs["input_ids"].astype(np.int64),
    "attention_mask": inputs["attention_mask"].astype(np.int64),
    "token_type_ids": inputs["token_type_ids"].astype(np.int64),
})

for i, text in enumerate(texts):
    predictions = np.argmax(outputs[0][i], axis=-1)
    tokens = tokenizer.convert_ids_to_tokens(inputs["input_ids"][i])
    # ... process as above

Quantization Details

The model was quantized from the original fp32 ONNX export using ONNX Runtime's MatMulNBitsQuantizer:

from onnxruntime.quantization.matmul_nbits_quantizer import MatMulNBitsQuantizer
import onnx

model = onnx.load("model.onnx")
quant = MatMulNBitsQuantizer(
    model=model,
    block_size=128,
    is_symmetric=True,
    accuracy_level=4,
    bits=4,
)
quant.process()

Citation

If you use this model, please cite the original LER dataset paper:

@inproceedings{leitner2020dataset,
  title={A Dataset of German Legal Documents for Named Entity Recognition},
  author={Leitner, Elena and Rehm, Georg and Moreno-Schneider, Juli{\'a}n},
  booktitle={Proceedings of the 12th Language Resources and Evaluation Conference},
  pages={4886--4893},
  year={2020},
  url={https://arxiv.org/abs/2003.13016}
}

Acknowledgments

• Source model by Elena Nereiss
• Dataset: German LER (CC-BY-4.0)
• Base model: bert-base-german-cased by deepset