268 lines
11 KiB
Python
268 lines
11 KiB
Python
from trytond.model import ModelSQL, ModelView, fields, Workflow
|
|
from trytond.pool import Pool, PoolMeta
|
|
from trytond.pyson import Eval
|
|
from trytond.wizard import Button
|
|
from trytond.transaction import Transaction
|
|
from sql import Table
|
|
from decimal import getcontext, Decimal, ROUND_HALF_UP
|
|
import requests
|
|
import io
|
|
import logging
|
|
import json
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
class AutomationDocument(ModelSQL, ModelView, Workflow):
|
|
"""Automation Document"""
|
|
__name__ = 'automation.document'
|
|
|
|
document = fields.Many2One('document.incoming', 'Document')
|
|
type = fields.Selection([
|
|
('invoice', 'Invoice'),
|
|
('statement_of_facts', 'Statement of Facts'),
|
|
('weight_report', 'Weight Report'),
|
|
('controller', 'Controller'),
|
|
('bol', 'Bill of Lading'),
|
|
('controller_invoice', 'Controller Invoice'),
|
|
], 'Type')
|
|
|
|
state = fields.Selection([
|
|
('draft', 'Draft'),
|
|
('ocr_done', 'OCR Done'),
|
|
('structure_done', 'Structure Done'),
|
|
('table_done', 'Table Done'),
|
|
('metadata_done', 'Metadata Done'),
|
|
('validated', 'Validated'),
|
|
('error', 'Error'),
|
|
], 'State', required=True)
|
|
|
|
ocr_text = fields.Text('OCR Text')
|
|
structure_json = fields.Text('Structure JSON')
|
|
tables_json = fields.Text('Tables JSON')
|
|
metadata_json = fields.Text('Metadata JSON')
|
|
notes = fields.Text('Notes')
|
|
rule_set = fields.Many2One('automation.rule.set', 'Rule Set')
|
|
|
|
@classmethod
|
|
def __setup__(cls):
|
|
super().__setup__()
|
|
cls._buttons.update({
|
|
'run_pipeline': {'invisible': Eval('state') == 'test', 'depends': ['state']},
|
|
'run_ocr': {'invisible': Eval('state') == 'test', 'depends': ['state']},
|
|
'run_structure': {'invisible': Eval('state') == 'test', 'depends': ['state']},
|
|
'run_tables': {'invisible': Eval('state') == 'test', 'depends': ['state']},
|
|
'run_metadata': {'invisible': Eval('state') == 'test', 'depends': ['state']},
|
|
})
|
|
|
|
# -------------------------------------------------------
|
|
# OCR
|
|
# -------------------------------------------------------
|
|
@classmethod
|
|
@ModelView.button
|
|
def run_ocr(cls, docs):
|
|
for doc in docs:
|
|
try:
|
|
if doc.type == 'weight_report':
|
|
# Décoder le fichier depuis le champ Binary
|
|
file_data = doc.document.data or b""
|
|
logger.info(f"File size: {len(file_data)} bytes")
|
|
logger.info(f"First 20 bytes: {file_data[:20]}")
|
|
logger.info(f"Last 20 bytes: {file_data[-20:]}")
|
|
|
|
file_name = doc.document.name or "document"
|
|
|
|
# Envoyer le fichier au service OCR
|
|
response = requests.post(
|
|
"http://automation-service:8006/ocr",
|
|
files={"file": (file_name, io.BytesIO(file_data))}
|
|
)
|
|
response.raise_for_status()
|
|
data = response.json()
|
|
logger.info("RUN_OCR_RESPONSE:%s",data)
|
|
doc.ocr_text = data.get("ocr_text", "")
|
|
doc.state = "ocr_done"
|
|
doc.notes = (doc.notes or "") + "OCR done\n"
|
|
else:
|
|
doc.ocr_text = (file_data or b'').decode('utf-8', errors='replace')
|
|
|
|
except Exception as e:
|
|
doc.state = "error"
|
|
doc.notes = (doc.notes or "") + f"OCR error: {e}\n"
|
|
doc.save()
|
|
# -------------------------------------------------------
|
|
# STRUCTURE (doctr)
|
|
# -------------------------------------------------------
|
|
@classmethod
|
|
@ModelView.button
|
|
def run_structure(cls, docs):
|
|
for doc in docs:
|
|
try:
|
|
file_data = doc.document.data or b""
|
|
logger.info(f"File size: {len(file_data)} bytes")
|
|
logger.info(f"First 20 bytes: {file_data[:20]}")
|
|
logger.info(f"Last 20 bytes: {file_data[-20:]}")
|
|
|
|
file_name = doc.document.name or "document"
|
|
|
|
response = requests.post(
|
|
"http://automation-service:8006/structure",
|
|
files={"file": (file_name, io.BytesIO(file_data))}
|
|
)
|
|
response.raise_for_status()
|
|
data = response.json()
|
|
|
|
doc.structure_json = data.get("structure", "")
|
|
doc.state = "structure_done"
|
|
doc.notes = (doc.notes or "") + "Structure parsing done\n"
|
|
|
|
except Exception as e:
|
|
doc.state = "error"
|
|
doc.notes = (doc.notes or "") + f"Structure error: {e}\n"
|
|
doc.save()
|
|
|
|
# -------------------------------------------------------
|
|
# TABLES (camelot)
|
|
# -------------------------------------------------------
|
|
@classmethod
|
|
@ModelView.button
|
|
def run_tables(cls, docs):
|
|
for doc in docs:
|
|
try:
|
|
file_data = doc.document.data or b""
|
|
logger.info(f"File size: {len(file_data)} bytes")
|
|
logger.info(f"First 20 bytes: {file_data[:20]}")
|
|
logger.info(f"Last 20 bytes: {file_data[-20:]}")
|
|
|
|
file_name = doc.document.name or "document"
|
|
|
|
response = requests.post(
|
|
"http://automation-service:8006/tables",
|
|
files={"file": (file_name, io.BytesIO(file_data))}
|
|
)
|
|
response.raise_for_status()
|
|
data = response.json()
|
|
|
|
doc.tables_json = data.get("tables", "")
|
|
doc.state = "table_done"
|
|
doc.notes = (doc.notes or "") + "Table extraction done\n"
|
|
|
|
except Exception as e:
|
|
doc.state = "error"
|
|
doc.notes = (doc.notes or "") + f"Table error: {e}\n"
|
|
doc.save()
|
|
|
|
# -------------------------------------------------------
|
|
# METADATA (spaCy)
|
|
# -------------------------------------------------------
|
|
@classmethod
|
|
@ModelView.button
|
|
def run_metadata(cls, docs):
|
|
for doc in docs:
|
|
try:
|
|
logger.info("Sending OCR text to metadata API: %s", doc.ocr_text)
|
|
|
|
response = requests.post(
|
|
#"http://automation-service:8006/metadata",
|
|
"http://automation-service:8006/parse",
|
|
json={"text": doc.ocr_text or ""}
|
|
)
|
|
response.raise_for_status()
|
|
data = response.json()
|
|
|
|
# Stocker le JSON complet renvoyé par l'API
|
|
#doc.metadata_json = data
|
|
doc.metadata_json = json.dumps(data, indent=4, ensure_ascii=False)
|
|
doc.state = "metadata_done"
|
|
doc.notes = (doc.notes or "") + "Metadata extraction done\n"
|
|
|
|
except requests.exceptions.RequestException as e:
|
|
doc.state = "error"
|
|
doc.notes = (doc.notes or "") + f"Metadata HTTP error: {e}\n"
|
|
logger.error("Metadata HTTP error: %s", e)
|
|
except Exception as e:
|
|
doc.state = "error"
|
|
doc.notes = (doc.notes or "") + f"Metadata processing error: {e}\n"
|
|
logger.error("Metadata processing error: %s", e)
|
|
|
|
doc.save()
|
|
|
|
def create_weight_report(self,wr_payload):
|
|
|
|
response = requests.post(
|
|
"http://automation-service:8006/weight-report",
|
|
json=wr_payload, # 👈 ICI la correction
|
|
timeout=10
|
|
)
|
|
|
|
response.raise_for_status()
|
|
return response.json()
|
|
|
|
# -------------------------------------------------------
|
|
# FULL PIPELINE
|
|
# -------------------------------------------------------
|
|
@classmethod
|
|
@ModelView.button
|
|
def run_pipeline(cls, docs):
|
|
for doc in docs:
|
|
try:
|
|
logger.info("DATA_TYPE:%s",type(doc.metadata_json))
|
|
metadata = json.loads(str(doc.metadata_json))
|
|
logger.info("JSON STRUCTURE:%s",metadata)
|
|
|
|
WeightReport = Pool().get('weight.report')
|
|
wr = WeightReport.create_from_json(metadata)
|
|
|
|
ShipmentIn = Pool().get('stock.shipment.in')
|
|
ShipmentWR = Pool().get('shipment.wr')
|
|
sh = ShipmentIn.search([('bl_number','ilike',wr.bl_no)])
|
|
if sh:
|
|
swr = ShipmentWR()
|
|
swr.shipment_in = sh[0]
|
|
swr.wr = wr
|
|
ShipmentWR.save([swr])
|
|
doc.notes = (doc.notes or "") + f"Shipment found: {sh[0].number}\n"
|
|
logger.info("BL_NUMBER:%s",sh[0].bl_number)
|
|
sale_line = sh[0]._create_lots_from_fintrade()
|
|
if sale_line:
|
|
logger.info("CREATE_FINTRADE_WR_FOR_SL:%s",sale_line)
|
|
weight_total = sum([l.lot_quantity for l in sale_line.lots if l.lot_type == 'physic'])
|
|
factor = weight_total / wr.net_landed_kg if wr.net_landed_kg else 1
|
|
for lot in sale_line.lots:
|
|
if lot.lot_type == 'physic':
|
|
wr_payload = {
|
|
"chunk_key": lot.lot_chunk_key,
|
|
"gross_weight": float(round(lot.lot_gross_quantity / factor,5)),
|
|
"net_weight": float(round(lot.lot_quantity / factor,5)),
|
|
"tare_total": float(round(wr.tare_kg * (lot.lot_quantity / weight_total),5)) ,
|
|
"bags": int(wr.bales * (lot.lot_quantity / weight_total)),
|
|
"surveyor_code": 231,
|
|
"place_key": 0,
|
|
"report_date": 20260127
|
|
}
|
|
logger.info("PAYLOAD:%s",wr_payload)
|
|
data = doc.create_weight_report(wr_payload)
|
|
doc.notes = (doc.notes or "") + f"WR created in Fintrade: {data.get('success')}\n"
|
|
doc.notes = (doc.notes or "") + f"WR key: {data.get('weight_report_key')}\n"
|
|
|
|
# if cls.rule_set.ocr_required:[]
|
|
# cls.run_ocr([doc])
|
|
# if cls.rule_set.structure_required and doc.state != "error":
|
|
# cls.run_structure([doc])
|
|
# if cls.rule_set.table_required and doc.state != "error":
|
|
# cls.run_tables([doc])
|
|
# if cls.rule_set.metadata_required and doc.state != "error":
|
|
# cls.run_metadata([doc])
|
|
# if doc.state != "error":
|
|
# doc.state = "validated"
|
|
# doc.notes = (doc.notes or "") + "Pipeline completed\n"
|
|
except Exception as e:
|
|
logger.exception("PIPELINE FAILED") # 👈 TRACE COMPLETE
|
|
doc.state = "error"
|
|
doc.notes = (doc.notes or "") + f"Pipeline error: {e}\n"
|
|
doc.save()
|
|
raise
|
|
|
|
# except Exception as e:
|
|
# doc.state = "error"
|
|
# doc.notes = (doc.notes or "") + f"Pipeline error: {e}\n"
|
|
doc.save() |