Faircot cron

modules/automation/cron.py

@@ -348,15 +348,23 @@ class AutomationCron(ModelSQL, ModelView):
                     with Transaction().set_context(
                             _purchase_trade_skip_physical_lot_tolerance_warning=True):
                         inv_date, inv_nb = shipment._create_lots_from_fintrade()
-                    shipment.controller = shipment.get_controller()
+                    shipment = ShipmentIn(shipment.id)
-                    shipment.controller_target = controller
+                    if shipment.incoming_moves:
-                    shipment.create_fee(shipment.controller)
+                        shipment.controller = shipment.get_controller()
-                    shipment.instructions = shipment.get_instructions_html(
+                        shipment.controller_target = controller
-                        inv_date, inv_nb)
+                        shipment.create_fee(shipment.controller)
-                    ShipmentIn.save([shipment])
+                        shipment.instructions = shipment.get_instructions_html(
+                            inv_date, inv_nb)
+                        ShipmentIn.save([shipment])
+                    else:
+                        logger.info(
+                            "Shipment %s cree sans lots : fees/instructions "
+                            "non crees, voir FINTRADE_CLEANUP_NEEDED",
+                            si_number)
                     trans_shipment.commit()
-                    successful_shipments += 1
+                    if shipment.incoming_moves:
-                    logger.info(f"Shipment {si_number} cree avec succes")
+                        successful_shipments += 1
+                        logger.info(f"Shipment {si_number} cree avec succes")
 
             except Exception as e:
                 error_details = {

modules/purchase_trade/lot.py

@@ -1015,7 +1015,6 @@ class Lot(metaclass=PoolMeta):
                 lqt.lot_s = lot_s
                 lqt.lot_p = None
         if sh:
-            logger.info("IDSHIP:%s",sh)
             str_sh = str(sh)
             index = str_sh.rfind(",")
             if index != -1:
@@ -1060,8 +1059,6 @@ class Lot(metaclass=PoolMeta):
         else:
             lqts = LotQt.search([('lot_p','=',vlot),('lot_s','=',lot_s),('lot_shipment_in','=',None),('lot_shipment_internal','=',None),('lot_shipment_out','=',None)])
         if lqts:
-            logger.info("UVP_QT:%s",qt)
-            logger.info("UVP_LQ:%s",lqts[0].lot_quantity)
             lqts[0].lot_quantity += qt
             if lqts[0].lot_quantity < 0:
                 lqts[0].lot_quantity = 0

modules/purchase_trade/stock.py

@@ -2427,13 +2427,91 @@ class ShipmentIn(metaclass=PoolMeta):
             "FINTRADE_LOTS_START shipment=%s reference=%s bl=%s rows=%s",
             getattr(self, 'id', None), self.reference, self.bl_number,
             len(rows))
-        logger.info("ROWS:%s",rows)
         inv_date = None
         inv_nb = None
+        cleanup_needed = []
+        skipped_rows = 0
+
+        def normalize_quantity(value):
+            return Decimal(str(value or 0)).quantize(Decimal('0.00001'))
+
+        def lot_summary(lot):
+            return {
+                'id': lot.id,
+                'name': getattr(lot, 'lot_name', None),
+                'shipment_in': (
+                    lot.lot_shipment_in.id
+                    if getattr(lot, 'lot_shipment_in', None)
+                    else None),
+                'purchase_line': (
+                    lot.line.id
+                    if getattr(lot, 'line', None)
+                    else None),
+                'sale_line': (
+                    lot.sale_line.id
+                    if getattr(lot, 'sale_line', None)
+                    else None),
+            }
+
+        def lqt_summary(lqt):
+            return {
+                'id': lqt.id,
+                'lot_p': lqt.lot_p.id if lqt.lot_p else None,
+                'lot_s': lqt.lot_s.id if lqt.lot_s else None,
+                'quantity': lqt.lot_quantity,
+                'unit': lqt.lot_unit.id if lqt.lot_unit else None,
+                'shipment_in': (
+                    lqt.lot_shipment_in.id
+                    if lqt.lot_shipment_in else None),
+                'shipment_origin': lqt.lot_shipment_origin,
+                'status': lqt.lot_status,
+                'availability': lqt.lot_av,
+            }
+
+        def add_cleanup(reason, declaration, chunk, lot_number, quantity,
+                details=None):
+            item = {
+                'reason': reason,
+                'shipment': getattr(self, 'id', None),
+                'reference': self.reference,
+                'bl': self.bl_number,
+                'declaration': declaration,
+                'chunk': chunk,
+                'lot_number': lot_number,
+                'quantity': quantity,
+                'details': details or {},
+            }
+            cleanup_needed.append(item)
+            logger.info("FINTRADE_CLEANUP_NEEDED %s", item)
+
+        def select_or_link_lqt(lqts, quantity):
+            quantity = normalize_quantity(quantity)
+            current = [
+                q for q in lqts
+                if q.lot_p
+                and q.lot_shipment_in
+                and q.lot_shipment_in.id == self.id
+                and normalize_quantity(q.lot_quantity) >= quantity
+            ]
+            if current:
+                return current[0]
+            unshipped = [
+                q for q in lqts
+                if q.lot_p
+                and not q.lot_shipment_in
+                and not q.lot_shipment_internal
+                and not q.lot_shipment_out
+                and normalize_quantity(q.lot_quantity) >= quantity
+            ]
+            if not unshipped:
+                return None
+            return LotQt.link_to_transport(
+                unshipped[0], quantity,
+                'stock.shipment.in,%s' % self.id)
+
         if rows:
             sale_line = None
             for row in rows:
-                logger.info("ROW:%s",row)
                 #Purchase & Sale creation
                 LotQt = Pool().get('lot.qt')
                 Lot = Pool().get('lot.lot')
@@ -2459,7 +2537,6 @@ class ShipmentIn(metaclass=PoolMeta):
                 lot_unit = str(row[5]).strip().lower()
                 product = str(row[6]).strip().upper()
                 lot_net_weight = Decimal(row[4])
-                logger.info("LOT_NET_WEIGHT:%s",lot_net_weight)
                 lot_gross_weight = Decimal(row[3])
                 lot_bales = Decimal(row[2])
                 lot_number = row[1] 
@@ -2480,33 +2557,49 @@ class ShipmentIn(metaclass=PoolMeta):
                     chunk_key, dec_key, lot_net_weight, lot_gross_weight,
                     lot_bales, lot_unit, sell_price_currency,
                     sell_price_unit, customer, reference)
+                if chunk_key_int is None:
+                    skipped_rows += 1
+                    add_cleanup(
+                        'invalid_chunk_key', dec_key, chunk_key, lot_number,
+                        lot_net_weight)
+                    continue
+
                 if chunk_key_int is not None:
                     existing_chunk_lots = Lot.search([
                         ('lot_chunk_key', '=', chunk_key_int),
                     ])
+                    if existing_chunk_lots:
+                        duplicate_details = {
+                            'existing_count': len(existing_chunk_lots),
+                            'existing_sample': [
+                                lot_summary(lot)
+                                for lot in existing_chunk_lots[:20]
+                            ],
+                            'existing_shipments': sorted({
+                                lot.lot_shipment_in.id
+                                for lot in existing_chunk_lots
+                                if getattr(lot, 'lot_shipment_in', None)
+                            }),
+                        }
+                        skipped_rows += 1
+                        add_cleanup(
+                            'chunk_already_has_physical_lots',
+                            dec_key, chunk_key, lot_number,
+                            lot_net_weight, duplicate_details)
+                        logger.info(
+                            "FINTRADE_SKIP_DUPLICATE_CHUNK shipment=%s "
+                            "reference=%s chunk=%s existing_count=%s "
+                            "existing_shipments=%s",
+                            getattr(self, 'id', None), self.reference,
+                            chunk_key, duplicate_details['existing_count'],
+                            duplicate_details['existing_shipments'])
+                        continue
                     logger.info(
-                        "FINTRADE_CHUNK_CHECK chunk=%s existing_lots=%s",
+                        "FINTRADE_CHUNK_CHECK chunk=%s existing_count=%s "
+                        "existing_sample=%s",
                         chunk_key,
-                        [
+                        len(existing_chunk_lots),
-                            {
+                        [lot_summary(lot) for lot in existing_chunk_lots[:20]])
-                                'id': lot.id,
-                                'name': getattr(lot, 'lot_name', None),
-                                'shipment_in': (
-                                    lot.lot_shipment_in.id
-                                    if getattr(lot, 'lot_shipment_in', None)
-                                    else None),
-                                'purchase_line': (
-                                    lot.line.id
-                                    if getattr(lot, 'line', None)
-                                    else None),
-                                'sale_line': (
-                                    lot.sale_line.id
-                                    if getattr(lot, 'sale_line', None)
-                                    else None),
-                            }
-                            for lot in existing_chunk_lots
-                        ])
-                logger.info("DECLARATION_KEY:%s",dec_key)
                 declaration = SaleLine.search(['note','=',dec_key])
                 logger.info(
                     "FINTRADE_DECLARATION_SEARCH declaration=%s count=%s "
@@ -2522,37 +2615,40 @@ class ShipmentIn(metaclass=PoolMeta):
                         sale_line.sale.id if sale_line.sale else None,
                         sale_line.quantity, sale_line.quantity_theorical,
                         [lot.id for lot in sale_line.lots])
+                    if not sale_line.lots:
+                        skipped_rows += 1
+                        add_cleanup(
+                            'declaration_without_virtual_lot',
+                            dec_key, chunk_key, lot_number, lot_net_weight,
+                            {'sale_line': sale_line.id})
+                        continue
                     vlot = sale_line.lots[0]
                     lqt = LotQt.search([('lot_s','=',vlot.id)])
                     logger.info(
                         "FINTRADE_DECLARATION_LQT declaration=%s vlot=%s "
                         "vlot_qty=%s lqt_count=%s lqts=%s",
                         dec_key, vlot.id, vlot.lot_quantity, len(lqt),
-                        [
+                        [lqt_summary(q) for q in lqt])
+                    lqt = select_or_link_lqt(lqt, lot_net_weight)
+                    if not lqt:
+                        skipped_rows += 1
+                        add_cleanup(
+                            'no_open_matched_lot_qt_for_declaration',
+                            dec_key, chunk_key, lot_number, lot_net_weight,
                             {
-                                'id': q.id,
+                                'sale_line': sale_line.id,
-                                'lot_p': q.lot_p.id if q.lot_p else None,
+                                'vlot': vlot.id,
-                                'lot_s': q.lot_s.id if q.lot_s else None,
+                                'available_lqts': [
-                                'quantity': q.lot_quantity,
+                                    lqt_summary(q)
-                                'unit': q.lot_unit.id if q.lot_unit else None,
+                                    for q in LotQt.search([
-                                'shipment_in': (
+                                        ('lot_s', '=', vlot.id),
-                                    q.lot_shipment_in.id
+                                        ('lot_p', '>', 0),
-                                    if q.lot_shipment_in else None),
+                                    ])
-                                'shipment_origin': q.lot_shipment_origin,
+                                ],
-                                'status': q.lot_status,
+                            })
-                                'availability': q.lot_av,
+                        continue
-                            }
+                    sale_line.quantity_theorical += round(lot_net_weight,2)
-                            for q in lqt
+                    SaleLine.save([sale_line])
-                        ])
-                    if lqt:
-                        for lq in lqt:
-                            if lq.lot_p:
-                                logger.info("VLOT_P:%s",lq.lot_p)
-                                sale_line.quantity_theorical += round(lot_net_weight,2) 
-                                SaleLine.save([sale_line])
-                                lq.lot_p.updateVirtualPart(round(lot_net_weight,2),self,lq.lot_s)
-                                vlot.set_current_quantity(round(lot_net_weight,2),round(lot_gross_weight,2),1)
-                                Lot.save([vlot])
                 else:
                     sale = Sale()
                     sale_line = SaleLine()
@@ -2628,6 +2724,7 @@ class ShipmentIn(metaclass=PoolMeta):
                     ct.matched = True
                     ct.shipment_in = self
                     ct.lot = sale_line.lots[0]
+                    ct.quantity = sale_line.quantity
                     ct.product = sale_line.product
                     ct.unit = sale_line.unit
                     d.party = Party.getPartyByName('FAIRCOT')
@@ -2669,31 +2766,20 @@ class ShipmentIn(metaclass=PoolMeta):
             
                 #Lots creation
                 vlot = sale_line.lots[0]
-                lqt = LotQt.search([('lot_s','=',vlot.id),('lot_p','>',0)])
+                lqts = LotQt.search([
+                    ('lot_s', '=', vlot.id),
+                    ('lot_p', '>', 0),
+                ])
+                if not declaration:
+                    lqt = select_or_link_lqt(lqts, lot_net_weight)
                 logger.info(
                     "FINTRADE_PHYSICAL_LOT_SOURCE shipment=%s declaration=%s "
                     "chunk=%s vlot=%s vlot_qty=%s source_lqt_count=%s "
                     "source_lqts=%s",
                     getattr(self, 'id', None), dec_key, chunk_key, vlot.id,
-                    vlot.lot_quantity, len(lqt),
+                    vlot.lot_quantity, len(lqts),
-                    [
+                    [lqt_summary(q) for q in lqts])
-                        {
+                if lqt and normalize_quantity(lqt.lot_quantity) >= normalize_quantity(lot_net_weight):
-                            'id': q.id,
-                            'lot_p': q.lot_p.id if q.lot_p else None,
-                            'lot_s': q.lot_s.id if q.lot_s else None,
-                            'quantity': q.lot_quantity,
-                            'unit': q.lot_unit.id if q.lot_unit else None,
-                            'shipment_in': (
-                                q.lot_shipment_in.id
-                                if q.lot_shipment_in else None),
-                            'shipment_origin': q.lot_shipment_origin,
-                            'status': q.lot_status,
-                            'availability': q.lot_av,
-                        }
-                        for q in lqt
-                    ])
-                if lqt and vlot.lot_quantity > 0:
-                    lqt = lqt[0]
                     l = LotAdd()
                     l.lot_qt = lot_bales
                     l.lot_unit = Uom.get_by_name('bale')
@@ -2710,7 +2796,6 @@ class ShipmentIn(metaclass=PoolMeta):
                         getattr(self, 'id', None), dec_key, chunk_key,
                         lqt.id, lqt.lot_quantity, l.lot_quantity,
                         l.lot_gross_quantity, l.lot_qt)
-                    logger.info("ADD_LOT:%s",int(chunk_key))
                     LotQt.add_physical_lots(lqt,[l])
                     logger.info(
                         "FINTRADE_ADD_PHYSICAL_LOT_DONE shipment=%s "
@@ -2718,14 +2803,29 @@ class ShipmentIn(metaclass=PoolMeta):
                         getattr(self, 'id', None), dec_key, chunk_key,
                         lqt.id)
                 else:
+                    skipped_rows += 1
+                    reason = 'no matched purchase lqt'
+                    if lqt:
+                        reason = 'matched purchase lqt open quantity too low'
+                    add_cleanup(
+                        reason, dec_key, chunk_key, lot_number, lot_net_weight,
+                        {
+                            'selected_lqt': lqt_summary(lqt) if lqt else None,
+                            'source_lqts': [lqt_summary(q) for q in lqts],
+                        })
                     logger.info(
                         "FINTRADE_SKIP_PHYSICAL_LOT shipment=%s "
                         "declaration=%s chunk=%s reason=%s vlot_qty=%s "
                         "source_lqt_count=%s",
                         getattr(self, 'id', None), dec_key, chunk_key,
-                        'no matched purchase lqt'
+                        reason, vlot.lot_quantity, len(lqts))
-                        if not lqt else 'vlot quantity not positive',
+
-                        vlot.lot_quantity, len(lqt))
+        if cleanup_needed:
+            logger.info(
+                "FINTRADE_CLEANUP_SUMMARY shipment=%s reference=%s "
+                "skipped_rows=%s cleanup_count=%s cleanup_items=%s",
+                getattr(self, 'id', None), self.reference, skipped_rows,
+                len(cleanup_needed), cleanup_needed)
 
         return inv_date,inv_nb