issue when trying to use trained layoutlmv3

Question

I’m trying to layout model I train, but I don’t know how to do it. I mixed a few idea off the internet to do it but when I run it the result are completely wrong. The boxes don’t match at all even though during the training I have an F1 score of 0.924242 and an accuracy score of 0.948276. I trained my model on a 80 image dataset.

I have been trying to use the layoutlmv3 model I have trained to use locally on my school project

If any one can help, it would be amazing, I am a beginner in ml. TKS A LOT

The code i have been trying :

model_name = "checkpoint-1000"
model = AutoModelForTokenClassification.from_pretrained(model_name) 
processor = AutoProcessor.from_pretrained("microsoft/layoutlmv3-base", apply_ocr=True)

id2label = {0: 'key_achats_marchandises', 1: 'key_actif_circulant', 2: 'key_actif_immobilise', 3: 'key_ca', 4: 'key_charges_sociales', 5: 'key_date_cloture', 6: 'key_dette', 7: 'key_disponibilites', 8: 'key_dotations_immobilisations', 9: 'key_impots', 10: 'key_passif_circulant', 11: 'key_resultat_exploitations', 12: 'key_rn', 13: 'key_salaire', 14: 'key_transports_expeditions',}

label2color = {"key_achats_marchandises": "blue", "key_actif_circulant": "green", "key_actif_immobilise": "orange", "key_ca": "red", "key_charges_sociales": "purple", "key_date_cloture": "cyan", "key_dette": "magenta", "key_disponibilites": "yellow", "key_dotations_immobilisations": "blue", "key_impots": "green", "key_passif_circulant": "orange", "key_resultat_exploitations": "red", "key_rn": "purple", "key_salaire": "cyan", "key_transports_expeditions": "magenta",}

def unnormalize_box(bbox, width, height): 
     return [ width * (bbox[0] / 1000), height * (bbox[1] / 1000), width * (bbox[2] / 1000), height * (bbox[3] / 1000), ]

def iob_to_label(label): 
      return label

def process_image(image): 
      image = Image.open(image).convert("RGB") print(type(image))       width, height = image.size
      encoding = processor(image, truncation=True,              return_offsets_mapping=True, return_tensors="pt")
      offset_mapping = encoding.pop('offset_mapping')

      outputs = model(**encoding)

      predictions = outputs.logits.argmax(-1).squeeze().tolist()
      token_boxes = encoding.bbox.squeeze().tolist()
      print(predictions)
      is_subword = np.array(offset_mapping.squeeze().tolist())[:,0] != 0
      true_predictions = [id2label[pred] for idx, pred in       enumerate(predictions) if not is_subword[idx]]
      true_boxes = [unnormalize_box(box, width, height) for idx, box in enumerate(token_boxes) if not is_subword[idx]]
      print(true_predictions)
      draw = ImageDraw.Draw(image)
      font = ImageFont.load_default()
      for prediction, box in zip(true_predictions, true_boxes):
          predicted_label = iob_to_label(prediction)
          draw.rectangle(box, outline=label2color[predicted_label])
          draw.text((box[0]+10, box[1]-10), text=predicted_label,       fill=label2color[predicted_label], font=font)

      return image
process_image("bilans_842953788_2019-02-28_C_2020-01-31_page_5.png")