319 lines
14 KiB
Python
319 lines
14 KiB
Python
from trytond.model import ModelSQL, ModelView, fields, Workflow
|
|
from trytond.pool import Pool, PoolMeta
|
|
from trytond.pyson import Eval
|
|
from trytond.wizard import Button
|
|
from trytond.transaction import Transaction
|
|
from sql import Table
|
|
from decimal import getcontext, Decimal, ROUND_HALF_UP
|
|
import requests
|
|
import io
|
|
import logging
|
|
import json
|
|
import re
|
|
import traceback
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
class AutomationDocument(ModelSQL, ModelView, Workflow):
|
|
"""Automation Document"""
|
|
__name__ = 'automation.document'
|
|
|
|
document = fields.Many2One('document.incoming', 'Document')
|
|
type = fields.Selection([
|
|
('invoice', 'Invoice'),
|
|
('statement_of_facts', 'Statement of Facts'),
|
|
('weight_report', 'Weight Report'),
|
|
('controller', 'Controller'),
|
|
('bol', 'Bill of Lading'),
|
|
('controller_invoice', 'Controller Invoice'),
|
|
], 'Type')
|
|
|
|
state = fields.Selection([
|
|
('draft', 'Draft'),
|
|
('ocr_done', 'OCR Done'),
|
|
('structure_done', 'Structure Done'),
|
|
('table_done', 'Table Done'),
|
|
('metadata_done', 'Metadata Done'),
|
|
('validated', 'Validated'),
|
|
('error', 'Error'),
|
|
], 'State', required=True)
|
|
|
|
ocr_text = fields.Text('OCR Text')
|
|
structure_json = fields.Text('Structure JSON')
|
|
tables_json = fields.Text('Tables JSON')
|
|
metadata_json = fields.Text('Metadata JSON')
|
|
notes = fields.Text('Notes')
|
|
rule_set = fields.Many2One('automation.rule.set', 'Rule Set')
|
|
|
|
@classmethod
|
|
def __setup__(cls):
|
|
super().__setup__()
|
|
cls._buttons.update({
|
|
'run_pipeline': {'invisible': Eval('state') == 'test', 'depends': ['state']},
|
|
'run_ocr': {'invisible': Eval('state') == 'test', 'depends': ['state']},
|
|
'run_structure': {'invisible': Eval('state') == 'test', 'depends': ['state']},
|
|
'run_tables': {'invisible': Eval('state') == 'test', 'depends': ['state']},
|
|
'run_metadata': {'invisible': Eval('state') == 'test', 'depends': ['state']},
|
|
})
|
|
|
|
# -------------------------------------------------------
|
|
# OCR
|
|
# -------------------------------------------------------
|
|
@classmethod
|
|
@ModelView.button
|
|
def run_ocr(cls, docs):
|
|
for doc in docs:
|
|
logger.info(
|
|
"RUN_OCR_START doc=%s type=%s state=%s incoming=%s",
|
|
getattr(doc, 'id', None), getattr(doc, 'type', None),
|
|
getattr(doc, 'state', None),
|
|
getattr(getattr(doc, 'document', None), 'id', None))
|
|
try:
|
|
if doc.type == 'weight_report':
|
|
# Décoder le fichier depuis le champ Binary
|
|
if not doc.document:
|
|
raise ValueError("No incoming document linked")
|
|
|
|
file_data = doc.document.data or b""
|
|
file_name = doc.document.name or "document"
|
|
url = "http://automation-service:8006/ocr"
|
|
logger.info(
|
|
"RUN_OCR_FILE doc=%s incoming=%s name=%s size=%s "
|
|
"first_20=%r last_20=%r",
|
|
getattr(doc, 'id', None), doc.document.id,
|
|
file_name, len(file_data), file_data[:20],
|
|
file_data[-20:])
|
|
|
|
if not file_data:
|
|
raise ValueError("Incoming document has no binary data")
|
|
|
|
# Envoyer le fichier au service OCR
|
|
logger.info(
|
|
"RUN_OCR_HTTP_REQUEST doc=%s url=%s file_name=%s "
|
|
"size=%s",
|
|
getattr(doc, 'id', None), url, file_name,
|
|
len(file_data))
|
|
response = requests.post(
|
|
url,
|
|
files={"file": (file_name, io.BytesIO(file_data))},
|
|
timeout=120,
|
|
)
|
|
logger.info(
|
|
"RUN_OCR_HTTP_RESPONSE doc=%s status=%s reason=%s "
|
|
"elapsed=%s content_type=%s body_start=%r",
|
|
getattr(doc, 'id', None), response.status_code,
|
|
response.reason, response.elapsed,
|
|
response.headers.get('content-type'),
|
|
response.text[:1000])
|
|
response.raise_for_status()
|
|
data = response.json()
|
|
ocr_text = data.get("ocr_text", "") or ""
|
|
logger.info(
|
|
"RUN_OCR_JSON doc=%s keys=%s ocr_text_len=%s "
|
|
"ocr_text_start=%r",
|
|
getattr(doc, 'id', None), list(data.keys()),
|
|
len(ocr_text), ocr_text[:500])
|
|
doc.ocr_text = ocr_text
|
|
doc.state = "ocr_done"
|
|
doc.notes = (doc.notes or "") + "OCR done\n"
|
|
else:
|
|
doc.ocr_text = (doc.document.data or b"").decode('utf-8', errors='replace')
|
|
match = re.search(r"\bID\s*:\s*(\d+)", doc.ocr_text)
|
|
if match:
|
|
request_id = match.group(1)
|
|
match = re.search(r"\bBL\s*number\s*:\s*([A-Za-z0-9_-]+)", doc.ocr_text, re.IGNORECASE)
|
|
if match:
|
|
bl_number = match.group(1)
|
|
ShipmentIn = Pool().get('stock.shipment.in')
|
|
sh = ShipmentIn.search(['bl_number','=',bl_number])
|
|
if sh:
|
|
sh[0].returned_id = request_id
|
|
ShipmentIn.save(sh)
|
|
doc.notes = (doc.notes or "") + "Id returned: " + request_id
|
|
|
|
so_payload = {
|
|
"ServiceOrderKey": sh[0].service_order_key,
|
|
"ID_Number": request_id
|
|
}
|
|
|
|
response = requests.post(
|
|
"http://automation-service:8006/service-order-update",
|
|
json=so_payload,
|
|
timeout=10
|
|
)
|
|
response.raise_for_status()
|
|
doc.notes = (doc.notes or "") + " SO updated"
|
|
|
|
except Exception as e:
|
|
logger.error(
|
|
"RUN_OCR_FAILED doc=%s type=%s error=%s traceback=%s",
|
|
getattr(doc, 'id', None), getattr(doc, 'type', None),
|
|
e, traceback.format_exc())
|
|
doc.state = "error"
|
|
doc.notes = (doc.notes or "") + f"OCR error: {e}\n"
|
|
doc.save()
|
|
logger.info(
|
|
"RUN_OCR_END doc=%s state=%s ocr_text_len=%s",
|
|
getattr(doc, 'id', None), getattr(doc, 'state', None),
|
|
len(getattr(doc, 'ocr_text', None) or ""))
|
|
# -------------------------------------------------------
|
|
# STRUCTURE (doctr)
|
|
# -------------------------------------------------------
|
|
@classmethod
|
|
@ModelView.button
|
|
def run_structure(cls, docs):
|
|
for doc in docs:
|
|
try:
|
|
file_data = doc.document.data or b""
|
|
logger.info(f"File size: {len(file_data)} bytes")
|
|
logger.info(f"First 20 bytes: {file_data[:20]}")
|
|
logger.info(f"Last 20 bytes: {file_data[-20:]}")
|
|
|
|
file_name = doc.document.name or "document"
|
|
|
|
response = requests.post(
|
|
"http://automation-service:8006/structure",
|
|
files={"file": (file_name, io.BytesIO(file_data))}
|
|
)
|
|
response.raise_for_status()
|
|
data = response.json()
|
|
|
|
doc.structure_json = data.get("structure", "")
|
|
doc.state = "structure_done"
|
|
doc.notes = (doc.notes or "") + "Structure parsing done\n"
|
|
|
|
except Exception as e:
|
|
doc.state = "error"
|
|
doc.notes = (doc.notes or "") + f"Structure error: {e}\n"
|
|
doc.save()
|
|
|
|
# -------------------------------------------------------
|
|
# TABLES (camelot)
|
|
# -------------------------------------------------------
|
|
@classmethod
|
|
@ModelView.button
|
|
def run_tables(cls, docs):
|
|
for doc in docs:
|
|
try:
|
|
file_data = doc.document.data or b""
|
|
logger.info(f"File size: {len(file_data)} bytes")
|
|
logger.info(f"First 20 bytes: {file_data[:20]}")
|
|
logger.info(f"Last 20 bytes: {file_data[-20:]}")
|
|
|
|
file_name = doc.document.name or "document"
|
|
|
|
response = requests.post(
|
|
"http://automation-service:8006/tables",
|
|
files={"file": (file_name, io.BytesIO(file_data))}
|
|
)
|
|
response.raise_for_status()
|
|
data = response.json()
|
|
|
|
doc.tables_json = data.get("tables", "")
|
|
doc.state = "table_done"
|
|
doc.notes = (doc.notes or "") + "Table extraction done\n"
|
|
|
|
except Exception as e:
|
|
doc.state = "error"
|
|
doc.notes = (doc.notes or "") + f"Table error: {e}\n"
|
|
doc.save()
|
|
|
|
# -------------------------------------------------------
|
|
# METADATA (spaCy)
|
|
# -------------------------------------------------------
|
|
@classmethod
|
|
@ModelView.button
|
|
def run_metadata(cls, docs):
|
|
for doc in docs:
|
|
try:
|
|
logger.info("Sending OCR text to metadata API: %s", doc.ocr_text)
|
|
|
|
response = requests.post(
|
|
#"http://automation-service:8006/metadata",
|
|
"http://automation-service:8006/parse",
|
|
json={"text": doc.ocr_text or ""}
|
|
)
|
|
response.raise_for_status()
|
|
data = response.json()
|
|
|
|
# Stocker le JSON complet renvoyé par l'API
|
|
#doc.metadata_json = data
|
|
doc.metadata_json = json.dumps(data, indent=4, ensure_ascii=False)
|
|
doc.state = "metadata_done"
|
|
doc.notes = (doc.notes or "") + "Metadata extraction done\n"
|
|
|
|
except requests.exceptions.RequestException as e:
|
|
doc.state = "error"
|
|
doc.notes = (doc.notes or "") + f"Metadata HTTP error: {e}\n"
|
|
logger.error("Metadata HTTP error: %s", e)
|
|
except Exception as e:
|
|
doc.state = "error"
|
|
doc.notes = (doc.notes or "") + f"Metadata processing error: {e}\n"
|
|
logger.error("Metadata processing error: %s", e)
|
|
|
|
doc.save()
|
|
|
|
def create_weight_report(self,wr_payload):
|
|
|
|
response = requests.post(
|
|
"http://automation-service:8006/weight-report",
|
|
json=wr_payload, # 👈 ICI la correction
|
|
timeout=10
|
|
)
|
|
|
|
response.raise_for_status()
|
|
return response.json()
|
|
|
|
# -------------------------------------------------------
|
|
# FULL PIPELINE
|
|
# -------------------------------------------------------
|
|
@classmethod
|
|
@ModelView.button
|
|
def run_pipeline(cls, docs):
|
|
for doc in docs:
|
|
try:
|
|
logger.info("DATA_TYPE:%s",type(doc.metadata_json))
|
|
metadata = json.loads(str(doc.metadata_json))
|
|
logger.info("JSON STRUCTURE:%s",metadata)
|
|
|
|
WeightReport = Pool().get('weight.report')
|
|
wr = WeightReport.create_from_json(metadata)
|
|
|
|
ShipmentIn = Pool().get('stock.shipment.in')
|
|
ShipmentWR = Pool().get('shipment.wr')
|
|
sh = ShipmentIn.search([('bl_number','ilike',wr.bl_no)])
|
|
if sh:
|
|
swr = ShipmentWR()
|
|
swr.shipment_in = sh[0]
|
|
swr.wr = wr
|
|
ShipmentWR.save([swr])
|
|
doc.notes = (doc.notes or "") + f"Shipment found: {sh[0].number}\n"
|
|
logger.info("BL_NUMBER:%s",sh[0].bl_number)
|
|
doc.notes = (
|
|
(doc.notes or "")
|
|
+ "Global WR linked to shipment. "
|
|
+ "Create remote lot WRs from the weight report form.\n")
|
|
|
|
# if cls.rule_set.ocr_required:[]
|
|
# cls.run_ocr([doc])
|
|
# if cls.rule_set.structure_required and doc.state != "error":
|
|
# cls.run_structure([doc])
|
|
# if cls.rule_set.table_required and doc.state != "error":
|
|
# cls.run_tables([doc])
|
|
# if cls.rule_set.metadata_required and doc.state != "error":
|
|
# cls.run_metadata([doc])
|
|
# if doc.state != "error":
|
|
# doc.state = "validated"
|
|
# doc.notes = (doc.notes or "") + "Pipeline completed\n"
|
|
except Exception as e:
|
|
logger.exception("PIPELINE FAILED") # 👈 TRACE COMPLETE
|
|
doc.state = "error"
|
|
doc.notes = (doc.notes or "") + f"Pipeline error: {e}\n"
|
|
doc.save()
|
|
raise
|
|
|
|
# except Exception as e:
|
|
# doc.state = "error"
|
|
# doc.notes = (doc.notes or "") + f"Pipeline error: {e}\n"
|
|
doc.save()
|