354 lines
16 KiB
Python
354 lines
16 KiB
Python
from trytond.model import ModelSQL, ModelView, fields, Workflow
|
|
from trytond.pool import Pool, PoolMeta
|
|
from trytond.pyson import Eval
|
|
from trytond.wizard import Button
|
|
from trytond.transaction import Transaction
|
|
from sql import Table
|
|
from decimal import getcontext, Decimal, ROUND_HALF_UP
|
|
import requests
|
|
import io
|
|
import logging
|
|
import json
|
|
from trytond.modules.purchase_trade.service import ContractFactory
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
class AutomationDocument(ModelSQL, ModelView, Workflow):
|
|
"""Automation Document"""
|
|
__name__ = 'automation.document'
|
|
|
|
document = fields.Many2One('document.incoming', 'Document')
|
|
type = fields.Selection([
|
|
('invoice', 'Invoice'),
|
|
('statement_of_facts', 'Statement of Facts'),
|
|
('weight_report', 'Weight Report'),
|
|
('bol', 'Bill of Lading'),
|
|
('controller_invoice', 'Controller Invoice'),
|
|
], 'Type')
|
|
|
|
state = fields.Selection([
|
|
('draft', 'Draft'),
|
|
('ocr_done', 'OCR Done'),
|
|
('structure_done', 'Structure Done'),
|
|
('table_done', 'Table Done'),
|
|
('metadata_done', 'Metadata Done'),
|
|
('validated', 'Validated'),
|
|
('error', 'Error'),
|
|
], 'State', required=True)
|
|
|
|
ocr_text = fields.Text('OCR Text')
|
|
structure_json = fields.Text('Structure JSON')
|
|
tables_json = fields.Text('Tables JSON')
|
|
metadata_json = fields.Text('Metadata JSON')
|
|
notes = fields.Text('Notes')
|
|
rule_set = fields.Many2One('automation.rule.set', 'Rule Set')
|
|
|
|
@classmethod
|
|
def __setup__(cls):
|
|
super().__setup__()
|
|
cls._buttons.update({
|
|
'run_pipeline': {'invisible': Eval('state') == 'test', 'depends': ['state']},
|
|
'run_ocr': {'invisible': Eval('state') == 'test', 'depends': ['state']},
|
|
'run_structure': {'invisible': Eval('state') == 'test', 'depends': ['state']},
|
|
'run_tables': {'invisible': Eval('state') == 'test', 'depends': ['state']},
|
|
'run_metadata': {'invisible': Eval('state') == 'test', 'depends': ['state']},
|
|
})
|
|
|
|
# -------------------------------------------------------
|
|
# OCR
|
|
# -------------------------------------------------------
|
|
@classmethod
|
|
@ModelView.button
|
|
def run_ocr(cls, docs):
|
|
for doc in docs:
|
|
try:
|
|
# Décoder le fichier depuis le champ Binary
|
|
file_data = doc.document.data or b""
|
|
logger.info(f"File size: {len(file_data)} bytes")
|
|
logger.info(f"First 20 bytes: {file_data[:20]}")
|
|
logger.info(f"Last 20 bytes: {file_data[-20:]}")
|
|
|
|
file_name = doc.document.name or "document"
|
|
|
|
# Envoyer le fichier au service OCR
|
|
response = requests.post(
|
|
"http://automation-service:8006/ocr",
|
|
files={"file": (file_name, io.BytesIO(file_data))}
|
|
)
|
|
response.raise_for_status()
|
|
data = response.json()
|
|
logger.info("RUN_OCR_RESPONSE:%s",data)
|
|
doc.ocr_text = data.get("ocr_text", "")
|
|
doc.state = "ocr_done"
|
|
doc.notes = (doc.notes or "") + "OCR done\n"
|
|
|
|
except Exception as e:
|
|
doc.state = "error"
|
|
doc.notes = (doc.notes or "") + f"OCR error: {e}\n"
|
|
doc.save()
|
|
# -------------------------------------------------------
|
|
# STRUCTURE (doctr)
|
|
# -------------------------------------------------------
|
|
@classmethod
|
|
@ModelView.button
|
|
def run_structure(cls, docs):
|
|
for doc in docs:
|
|
try:
|
|
file_data = doc.document.data or b""
|
|
logger.info(f"File size: {len(file_data)} bytes")
|
|
logger.info(f"First 20 bytes: {file_data[:20]}")
|
|
logger.info(f"Last 20 bytes: {file_data[-20:]}")
|
|
|
|
file_name = doc.document.name or "document"
|
|
|
|
response = requests.post(
|
|
"http://automation-service:8006/structure",
|
|
files={"file": (file_name, io.BytesIO(file_data))}
|
|
)
|
|
response.raise_for_status()
|
|
data = response.json()
|
|
|
|
doc.structure_json = data.get("structure", "")
|
|
doc.state = "structure_done"
|
|
doc.notes = (doc.notes or "") + "Structure parsing done\n"
|
|
|
|
except Exception as e:
|
|
doc.state = "error"
|
|
doc.notes = (doc.notes or "") + f"Structure error: {e}\n"
|
|
doc.save()
|
|
|
|
# -------------------------------------------------------
|
|
# TABLES (camelot)
|
|
# -------------------------------------------------------
|
|
@classmethod
|
|
@ModelView.button
|
|
def run_tables(cls, docs):
|
|
for doc in docs:
|
|
try:
|
|
file_data = doc.document.data or b""
|
|
logger.info(f"File size: {len(file_data)} bytes")
|
|
logger.info(f"First 20 bytes: {file_data[:20]}")
|
|
logger.info(f"Last 20 bytes: {file_data[-20:]}")
|
|
|
|
file_name = doc.document.name or "document"
|
|
|
|
response = requests.post(
|
|
"http://automation-service:8006/tables",
|
|
files={"file": (file_name, io.BytesIO(file_data))}
|
|
)
|
|
response.raise_for_status()
|
|
data = response.json()
|
|
|
|
doc.tables_json = data.get("tables", "")
|
|
doc.state = "table_done"
|
|
doc.notes = (doc.notes or "") + "Table extraction done\n"
|
|
|
|
except Exception as e:
|
|
doc.state = "error"
|
|
doc.notes = (doc.notes or "") + f"Table error: {e}\n"
|
|
doc.save()
|
|
|
|
# -------------------------------------------------------
|
|
# METADATA (spaCy)
|
|
# -------------------------------------------------------
|
|
@classmethod
|
|
@ModelView.button
|
|
def run_metadata(cls, docs):
|
|
for doc in docs:
|
|
try:
|
|
logger.info("Sending OCR text to metadata API: %s", doc.ocr_text)
|
|
|
|
response = requests.post(
|
|
#"http://automation-service:8006/metadata",
|
|
"http://automation-service:8006/parse",
|
|
json={"text": doc.ocr_text or ""}
|
|
)
|
|
response.raise_for_status()
|
|
data = response.json()
|
|
|
|
# Stocker le JSON complet renvoyé par l'API
|
|
#doc.metadata_json = data
|
|
doc.metadata_json = json.dumps(data, indent=4, ensure_ascii=False)
|
|
doc.state = "metadata_done"
|
|
doc.notes = (doc.notes or "") + "Metadata extraction done\n"
|
|
|
|
except requests.exceptions.RequestException as e:
|
|
doc.state = "error"
|
|
doc.notes = (doc.notes or "") + f"Metadata HTTP error: {e}\n"
|
|
logger.error("Metadata HTTP error: %s", e)
|
|
except Exception as e:
|
|
doc.state = "error"
|
|
doc.notes = (doc.notes or "") + f"Metadata processing error: {e}\n"
|
|
logger.error("Metadata processing error: %s", e)
|
|
|
|
doc.save()
|
|
# -------------------------------------------------------
|
|
# FULL PIPELINE
|
|
# -------------------------------------------------------
|
|
@classmethod
|
|
@ModelView.button
|
|
def run_pipeline(cls, docs):
|
|
for doc in docs:
|
|
try:
|
|
logger.info("DATA_TYPE:%s",type(doc.metadata_json))
|
|
metadata = json.loads(str(doc.metadata_json))
|
|
logger.info("JSON STRUCTURE:%s",metadata)
|
|
|
|
WeightReport = Pool().get('weight.report')
|
|
wr = WeightReport.create_from_json(metadata)
|
|
|
|
ShipmentIn = Pool().get('stock.shipment.in')
|
|
ShipmentWR = Pool().get('shipment.wr')
|
|
sh = ShipmentIn.search([('bl_number','ilike',wr.bl_no)])
|
|
if sh:
|
|
swr = ShipmentWR()
|
|
swr.shipment_in = sh[0]
|
|
swr.wr = wr
|
|
ShipmentWR.save([swr])
|
|
doc.notes = (doc.notes or "") + f"Shipment found: {sh[0].number}\n"
|
|
|
|
t = Table('freight_booking_lots')
|
|
cursor = Transaction().connection.cursor()
|
|
cursor.execute(*t.select(
|
|
t.BOOKING_NUMBER,
|
|
t.LOT_NUMBER,
|
|
t.LOT_NBR_BALES,
|
|
t.LOT_GROSS_WEIGHT,
|
|
t.LOT_NET_WEIGHT,
|
|
t.LOT_UOM,
|
|
t.LOT_QUALITY,
|
|
t.CUSTOMER,
|
|
t.SELL_PRICE_CURRENCY,
|
|
t.SELL_PRICE_UNIT,
|
|
t.SELL_PRICE,
|
|
t.SALE_INVOICE,
|
|
t.SELL_INV_AMOUNT,
|
|
t.SALE_INVOICE_DATE,
|
|
t.SELL_PREMIUM,
|
|
t.SALE_CONTRACT,
|
|
t.DECLARATION_KEY,
|
|
where=(t.BOOKING_NUMBER == sh[0].bl_number)
|
|
))
|
|
|
|
rows = cursor.fetchall()
|
|
if rows:
|
|
for row in rows:
|
|
#Purchase & Sale creation
|
|
LotQt = Pool().get('lot.qt')
|
|
LotAdd = Pool().get('lot.add.line')
|
|
Currency = Pool().get('currency.currency')
|
|
Product = Pool().get('product.product')
|
|
Party = Pool().get('party.party')
|
|
Uom = Pool().get('product.uom')
|
|
Sale = Pool().get('sale.sale')
|
|
SaleLine = Pool().get('sale.line')
|
|
|
|
dec_key = str(rows[0][16]).strip()
|
|
lot_unit = str(rows[0][5]).strip().lower()
|
|
product = str(rows[0][6]).strip().upper()
|
|
lot_net_weight = Decimal(rows[0][4])
|
|
lot_gross_weight = Decimal(rows[0][3])
|
|
lot_bales = int(rows[0][2])
|
|
lot_number = rows[0][1]
|
|
customer = str(rows[0][7]).strip().upper()
|
|
sell_price_currency = str(rows[0][8]).strip().upper()
|
|
sell_price_unit = str(rows[0][9]).strip().lower()
|
|
sell_price = Decimal(rows[0][10])
|
|
premium = Decimal(rows[0][14])
|
|
reference = Decimal(rows[0][15])
|
|
|
|
declaration = SaleLine.search(['note','=',dec_key])
|
|
if declaration:
|
|
sale_line = declaration[0]
|
|
logger.info("WITH_DEC:%s",sale_line)
|
|
vlot = sale_line.lots[0]
|
|
lqt = LotQt.search([('lot_s','=',vlot.id)])
|
|
lqt.lot_p.updateVirtualPart(lot_net_weight,sh,lqt.lot_s)
|
|
logger.info("WITH_DEC_LOT_NET:%s",lot_net_weight)
|
|
else:
|
|
sale = Sale()
|
|
sale_line = SaleLine()
|
|
sale.party = Party.getPartyByName(customer)
|
|
sale.reference = reference
|
|
if sale.party.addresses:
|
|
sale.invoice_address = sale.party.addresses[0]
|
|
sale.shipment_address = sale.party.addresses[0]
|
|
|
|
if sell_price_currency == 'USC':
|
|
sale.currency = Currency.get_by_name('USD')
|
|
sale_line.enable_linked_currency = True
|
|
sale_line.linked_currency = 1
|
|
sale_line.linked_unit = Uom.get_by_name(sell_price_unit)
|
|
sale_line.linked_price = sell_price
|
|
sale_line.unit_price = sale_line.get_price_linked_currency()
|
|
else:
|
|
sale.currency = Currency.get_by_name(sell_price_currency)
|
|
sale_line.unit_price = sell_price
|
|
sale_line.unit = Uom.get_by_name(sell_price_unit)
|
|
sale_line.premium = premium
|
|
Sale.save([sale])
|
|
sale_line.sale = sale.id
|
|
sale_line.quantity = lot_net_weight
|
|
sale_line.quantity_theorical = lot_net_weight
|
|
sale_line.product = Product.get_by_name('BRAZIL COTTON')
|
|
sale_line.unit = Uom.get_id_by_name(lot_unit)
|
|
sale_line.price_type = 'priced'
|
|
sale_line.created_by_code = False
|
|
sale_line.note = dec_key
|
|
SaleLine.save([sale_line])
|
|
|
|
ContractStart = Pool().get('contracts.start')
|
|
ContractDetail = Pool().get('contract.detail')
|
|
ct = ContractStart()
|
|
d = ContractDetail()
|
|
ct.type = 'Purchase'
|
|
ct.matched = True
|
|
ct.shipment_in = sh
|
|
ct.lot = sale_line.lots[0]
|
|
d.party = Party.getPartyByName('FAIRCOT')
|
|
if sale_line.enable_linked_currency:
|
|
d.currency_unit = str(sale_line.linked_currency.id) + '_' + str(sale_line.linked_unit.id)
|
|
else:
|
|
d.currency_unit = str(sale.currency.id) + '_' + str(sale_line.unit.id)
|
|
d.quantity = sale_line.quantity
|
|
d.unit = sale_line.unit
|
|
d.price = sale_line.unit_price
|
|
d.price_type = 'priced'
|
|
ct.contracts = [d]
|
|
ContractFactory.create_contracts(
|
|
ct.contracts,
|
|
type_=ct.type,
|
|
ct=ct,
|
|
)
|
|
|
|
#Lots creation
|
|
vlot = sale_line.lots[0]
|
|
lqt = LotQt.search([('lot_s','=',vlot.id)])
|
|
if lqt and vlot.lot_quantity > 0:
|
|
lqt = lqt[0]
|
|
l = LotAdd()
|
|
l.lot_qt = lot_bales
|
|
l.lot_unit = Uom.get_by_name('bale')
|
|
l.lot_unit_line = lot_unit
|
|
l.lot_quantity = lot_net_weight
|
|
l.lot_gross_quantity = lot_gross_weight
|
|
l.lot_premium = premium
|
|
LotQt.add_physical_lots(lqt,[l])
|
|
|
|
|
|
|
|
# if cls.rule_set.ocr_required:[]
|
|
# cls.run_ocr([doc])
|
|
# if cls.rule_set.structure_required and doc.state != "error":
|
|
# cls.run_structure([doc])
|
|
# if cls.rule_set.table_required and doc.state != "error":
|
|
# cls.run_tables([doc])
|
|
# if cls.rule_set.metadata_required and doc.state != "error":
|
|
# cls.run_metadata([doc])
|
|
# if doc.state != "error":
|
|
# doc.state = "validated"
|
|
# doc.notes = (doc.notes or "") + "Pipeline completed\n"
|
|
except Exception as e:
|
|
doc.state = "error"
|
|
doc.notes = (doc.notes or "") + f"Pipeline error: {e}\n"
|
|
doc.save() |