from trytond.model import ModelSQL, ModelView, fields, Workflow from trytond.pool import Pool, PoolMeta from trytond.pyson import Eval from trytond.wizard import Button from trytond.transaction import Transaction from sql import Table from decimal import getcontext, Decimal, ROUND_HALF_UP import requests import io import logging import json import re import traceback logger = logging.getLogger(__name__) class AutomationDocument(ModelSQL, ModelView, Workflow): """Automation Document""" __name__ = 'automation.document' document = fields.Many2One('document.incoming', 'Document') type = fields.Selection([ ('invoice', 'Invoice'), ('statement_of_facts', 'Statement of Facts'), ('weight_report', 'Weight Report'), ('controller', 'Controller'), ('bol', 'Bill of Lading'), ('controller_invoice', 'Controller Invoice'), ], 'Type') state = fields.Selection([ ('draft', 'Draft'), ('ocr_done', 'OCR Done'), ('structure_done', 'Structure Done'), ('table_done', 'Table Done'), ('metadata_done', 'Metadata Done'), ('validated', 'Validated'), ('error', 'Error'), ], 'State', required=True) ocr_text = fields.Text('OCR Text') structure_json = fields.Text('Structure JSON') tables_json = fields.Text('Tables JSON') metadata_json = fields.Text('Metadata JSON') notes = fields.Text('Notes') rule_set = fields.Many2One('automation.rule.set', 'Rule Set') @classmethod def __setup__(cls): super().__setup__() cls._buttons.update({ 'run_pipeline': {'invisible': Eval('state') == 'test', 'depends': ['state']}, 'run_ocr': {'invisible': Eval('state') == 'test', 'depends': ['state']}, 'run_structure': {'invisible': Eval('state') == 'test', 'depends': ['state']}, 'run_tables': {'invisible': Eval('state') == 'test', 'depends': ['state']}, 'run_metadata': {'invisible': Eval('state') == 'test', 'depends': ['state']}, }) # ------------------------------------------------------- # OCR # ------------------------------------------------------- @classmethod @ModelView.button def run_ocr(cls, docs): for doc in docs: logger.info( "RUN_OCR_START doc=%s type=%s state=%s incoming=%s", getattr(doc, 'id', None), getattr(doc, 'type', None), getattr(doc, 'state', None), getattr(getattr(doc, 'document', None), 'id', None)) try: if doc.type == 'weight_report': # Décoder le fichier depuis le champ Binary if not doc.document: raise ValueError("No incoming document linked") file_data = doc.document.data or b"" file_name = doc.document.name or "document" url = "http://automation-service:8006/ocr" logger.info( "RUN_OCR_FILE doc=%s incoming=%s name=%s size=%s " "first_20=%r last_20=%r", getattr(doc, 'id', None), doc.document.id, file_name, len(file_data), file_data[:20], file_data[-20:]) if not file_data: raise ValueError("Incoming document has no binary data") # Envoyer le fichier au service OCR logger.info( "RUN_OCR_HTTP_REQUEST doc=%s url=%s file_name=%s " "size=%s", getattr(doc, 'id', None), url, file_name, len(file_data)) response = requests.post( url, files={"file": (file_name, io.BytesIO(file_data))}, timeout=120, ) logger.info( "RUN_OCR_HTTP_RESPONSE doc=%s status=%s reason=%s " "elapsed=%s content_type=%s body_start=%r", getattr(doc, 'id', None), response.status_code, response.reason, response.elapsed, response.headers.get('content-type'), response.text[:1000]) response.raise_for_status() data = response.json() ocr_text = data.get("ocr_text", "") or "" logger.info( "RUN_OCR_JSON doc=%s keys=%s ocr_text_len=%s " "ocr_text_start=%r", getattr(doc, 'id', None), list(data.keys()), len(ocr_text), ocr_text[:500]) doc.ocr_text = ocr_text doc.state = "ocr_done" doc.notes = (doc.notes or "") + "OCR done\n" else: doc.ocr_text = (doc.document.data or b"").decode('utf-8', errors='replace') match = re.search(r"\bID\s*:\s*(\d+)", doc.ocr_text) if match: request_id = match.group(1) match = re.search(r"\bBL\s*number\s*:\s*([A-Za-z0-9_-]+)", doc.ocr_text, re.IGNORECASE) if match: bl_number = match.group(1) ShipmentIn = Pool().get('stock.shipment.in') sh = ShipmentIn.search(['bl_number','=',bl_number]) if sh: sh[0].returned_id = request_id ShipmentIn.save(sh) doc.notes = (doc.notes or "") + "Id returned: " + request_id so_payload = { "ServiceOrderKey": sh[0].service_order_key, "ID_Number": request_id } response = requests.post( "http://automation-service:8006/service-order-update", json=so_payload, timeout=10 ) response.raise_for_status() doc.notes = (doc.notes or "") + " SO updated" except Exception as e: logger.error( "RUN_OCR_FAILED doc=%s type=%s error=%s traceback=%s", getattr(doc, 'id', None), getattr(doc, 'type', None), e, traceback.format_exc()) doc.state = "error" doc.notes = (doc.notes or "") + f"OCR error: {e}\n" doc.save() logger.info( "RUN_OCR_END doc=%s state=%s ocr_text_len=%s", getattr(doc, 'id', None), getattr(doc, 'state', None), len(getattr(doc, 'ocr_text', None) or "")) # ------------------------------------------------------- # STRUCTURE (doctr) # ------------------------------------------------------- @classmethod @ModelView.button def run_structure(cls, docs): for doc in docs: try: file_data = doc.document.data or b"" logger.info(f"File size: {len(file_data)} bytes") logger.info(f"First 20 bytes: {file_data[:20]}") logger.info(f"Last 20 bytes: {file_data[-20:]}") file_name = doc.document.name or "document" response = requests.post( "http://automation-service:8006/structure", files={"file": (file_name, io.BytesIO(file_data))} ) response.raise_for_status() data = response.json() doc.structure_json = data.get("structure", "") doc.state = "structure_done" doc.notes = (doc.notes or "") + "Structure parsing done\n" except Exception as e: doc.state = "error" doc.notes = (doc.notes or "") + f"Structure error: {e}\n" doc.save() # ------------------------------------------------------- # TABLES (camelot) # ------------------------------------------------------- @classmethod @ModelView.button def run_tables(cls, docs): for doc in docs: try: file_data = doc.document.data or b"" logger.info(f"File size: {len(file_data)} bytes") logger.info(f"First 20 bytes: {file_data[:20]}") logger.info(f"Last 20 bytes: {file_data[-20:]}") file_name = doc.document.name or "document" response = requests.post( "http://automation-service:8006/tables", files={"file": (file_name, io.BytesIO(file_data))} ) response.raise_for_status() data = response.json() doc.tables_json = data.get("tables", "") doc.state = "table_done" doc.notes = (doc.notes or "") + "Table extraction done\n" except Exception as e: doc.state = "error" doc.notes = (doc.notes or "") + f"Table error: {e}\n" doc.save() # ------------------------------------------------------- # METADATA (spaCy) # ------------------------------------------------------- @classmethod @ModelView.button def run_metadata(cls, docs): for doc in docs: try: logger.info("Sending OCR text to metadata API: %s", doc.ocr_text) response = requests.post( #"http://automation-service:8006/metadata", "http://automation-service:8006/parse", json={"text": doc.ocr_text or ""} ) response.raise_for_status() data = response.json() # Stocker le JSON complet renvoyé par l'API #doc.metadata_json = data doc.metadata_json = json.dumps(data, indent=4, ensure_ascii=False) doc.state = "metadata_done" doc.notes = (doc.notes or "") + "Metadata extraction done\n" except requests.exceptions.RequestException as e: doc.state = "error" doc.notes = (doc.notes or "") + f"Metadata HTTP error: {e}\n" logger.error("Metadata HTTP error: %s", e) except Exception as e: doc.state = "error" doc.notes = (doc.notes or "") + f"Metadata processing error: {e}\n" logger.error("Metadata processing error: %s", e) doc.save() def create_weight_report(self,wr_payload): response = requests.post( "http://automation-service:8006/weight-report", json=wr_payload, # 👈 ICI la correction timeout=10 ) response.raise_for_status() return response.json() # ------------------------------------------------------- # FULL PIPELINE # ------------------------------------------------------- @classmethod @ModelView.button def run_pipeline(cls, docs): for doc in docs: try: logger.info("DATA_TYPE:%s",type(doc.metadata_json)) metadata = json.loads(str(doc.metadata_json)) logger.info("JSON STRUCTURE:%s",metadata) WeightReport = Pool().get('weight.report') wr = WeightReport.create_from_json(metadata) ShipmentIn = Pool().get('stock.shipment.in') ShipmentWR = Pool().get('shipment.wr') sh = ShipmentIn.search([('bl_number','ilike',wr.bl_no)]) if sh: swr = ShipmentWR() swr.shipment_in = sh[0] swr.wr = wr ShipmentWR.save([swr]) doc.notes = (doc.notes or "") + f"Shipment found: {sh[0].number}\n" logger.info("BL_NUMBER:%s",sh[0].bl_number) doc.notes = ( (doc.notes or "") + "Global WR linked to shipment. " + "Create remote lot WRs from the weight report form.\n") # if cls.rule_set.ocr_required:[] # cls.run_ocr([doc]) # if cls.rule_set.structure_required and doc.state != "error": # cls.run_structure([doc]) # if cls.rule_set.table_required and doc.state != "error": # cls.run_tables([doc]) # if cls.rule_set.metadata_required and doc.state != "error": # cls.run_metadata([doc]) # if doc.state != "error": # doc.state = "validated" # doc.notes = (doc.notes or "") + "Pipeline completed\n" except Exception as e: logger.exception("PIPELINE FAILED") # 👈 TRACE COMPLETE doc.state = "error" doc.notes = (doc.notes or "") + f"Pipeline error: {e}\n" doc.save() raise # except Exception as e: # doc.state = "error" # doc.notes = (doc.notes or "") + f"Pipeline error: {e}\n" doc.save()