ITSA - Import Prices

modules/purchase_trade/__init__.py

@@ -55,15 +55,18 @@ def register():
         lc.LCMT700,
         lc.LCMessage,
         lc.CreateLCStart,
-        global_reporting.GRConfiguration,
-        module='purchase_trade', type_='model')
+        global_reporting.GRConfiguration,
+        pricing.ImportPricesStart,
+        pricing.ImportPricesResult,
+        module='purchase_trade', type_='model')
     Pool.register(
-        incoming.ImportSwift,
-        incoming.PrepareDocuments,
-        incoming.AnalyzeConditions,
-        lc.CreateLCWizard,
-        module='purchase_trade', type_='wizard'
-    )
+        incoming.ImportSwift,
+        incoming.PrepareDocuments,
+        incoming.AnalyzeConditions,
+        lc.CreateLCWizard,
+        pricing.ImportPrices,
+        module='purchase_trade', type_='wizard'
+    )
     Pool.register(
         credit_risk.Party,
         credit_risk.CreditRiskRule,

modules/purchase_trade/docs/business-rules.md

@@ -155,3 +155,21 @@ Pour cette regle, couvrir au minimum:
 - augmentation sans `lot.qt` ouvert
 - diminution possible
 - diminution impossible avec erreur
+
+
+### BR-PT-004 - Market Price Import
+
+- Goal: import market prices from an Excel file
+- Description: after having selected an Excel file with the following columns:
+  - price_index: name of the market
+  - price_date: date of the price
+  - high price: high price
+  - low price: low price
+  - open price: open price
+  - price value: price value
+- Expected behavior:
+  - Check if price_index exists prior to impport related prices
+  - If price_index does not exist, check if option "Create price index is missing" has been checked in the input screen. If checked then try to create first the price index (table: price_price). If option is not checked, then ignore the price_index and mark it as "skipped" in the output results of the import.
+  - Check if price_date for the price_index already exists. 
+  - If price_date does not exist, then import the price. If price_date already exists then check if option "Overwrite existing price" is checked in the input screen. If checked, update the price for price_index and price_date. If option is not checked, then ignore the price_date and mark as "skipped" in the output.
+

modules/purchase_trade/pricing.py

@@ -1,9 +1,10 @@
 # This file is part of Tryton.  The COPYRIGHT file at the top level of
 # this repository contains the full copyright notices and license terms.
-from trytond.model import fields
-from trytond.pool import Pool, PoolMeta
-from trytond.pyson import Bool, Eval, Id
-from trytond.model import (ModelSQL, ModelView)
+from trytond.model import fields
+from trytond.exceptions import UserError
+from trytond.pool import Pool, PoolMeta
+from trytond.pyson import Bool, Eval, Id
+from trytond.model import (ModelSQL, ModelView)
 from trytond.tools import is_full_text, lstrip_wildcard
 from trytond.transaction import Transaction, inactive_records
 from decimal import getcontext, Decimal, ROUND_HALF_UP
@@ -11,12 +12,16 @@ from sql.aggregate import Count, Max, Min, Sum, Avg, BoolOr
 from sql.conditionals import Case
 from sql import Column, Literal
 from sql.functions import CurrentTimestamp, DateTrunc
-from trytond.wizard import Button, StateTransition, StateView, Wizard
-from itertools import chain, groupby
-from operator import itemgetter
-import datetime
-import logging
-from trytond.modules.purchase_trade.purchase import (TRIGGERS)
+from trytond.wizard import Button, StateTransition, StateView, Wizard
+from itertools import chain, groupby
+from operator import itemgetter
+import datetime
+import logging
+import re
+import zipfile
+from io import BytesIO
+from xml.etree import ElementTree
+from trytond.modules.purchase_trade.purchase import (TRIGGERS)
 
 logger = logging.getLogger(__name__)
 
@@ -50,13 +55,302 @@ DAYS = [
     ('sunday', 'Sunday'),
     ]
 
-class Estimated(ModelSQL, ModelView):
-    "Estimated date"
-    __name__ = 'pricing.estimated'
+class Estimated(ModelSQL, ModelView):
+    "Estimated date"
+    __name__ = 'pricing.estimated'
 
     trigger = fields.Selection(TRIGGERS,"Trigger")
-    estimated_date = fields.Date("Estimated date")
-    fin_int_delta = fields.Integer("Financing interests delta")
+    estimated_date = fields.Date("Estimated date")
+    fin_int_delta = fields.Integer("Financing interests delta")
+
+
+class ImportPricesStart(ModelView):
+    "Import Prices"
+    __name__ = 'purchase_trade.import_prices.start'
+
+    file_ = fields.Binary('Excel file', required=True, filename='filename')
+    filename = fields.Char('Filename')
+    create_missing_price_index = fields.Boolean(
+        "Create price index if missing")
+    overwrite_existing_price = fields.Boolean("Overwrite existing price")
+
+
+class ImportPricesResult(ModelView):
+    "Import Prices Result"
+    __name__ = 'purchase_trade.import_prices.result'
+
+    message = fields.Text("Results", readonly=True)
+
+
+class ImportPrices(Wizard):
+    "Import Prices"
+    __name__ = 'purchase_trade.import_prices'
+
+    start = StateView(
+        'purchase_trade.import_prices.start',
+        'purchase_trade.import_prices_start_view_form',
+        [
+            Button('Cancel', 'end', 'tryton-cancel'),
+            Button('Import', 'import_', 'tryton-ok', default=True),
+        ])
+    import_ = StateTransition()
+    result = StateView(
+        'purchase_trade.import_prices.result',
+        'purchase_trade.import_prices_result_view_form',
+        [
+            Button('OK', 'end', 'tryton-ok', default=True),
+        ])
+
+    REQUIRED_COLUMNS = {
+        'priceindex': 'price_index',
+        'pricedate': 'price_date',
+        'highprice': 'high_price',
+        'lowprice': 'low_price',
+        'openprice': 'open_price',
+        'pricevalue': 'price_value',
+    }
+
+    def transition_import_(self):
+        rows = self._read_xlsx(self.start.file_)
+        stats = self._import_rows(
+            rows,
+            create_missing_price_index=(
+                self.start.create_missing_price_index),
+            overwrite_existing_price=self.start.overwrite_existing_price)
+        self.result.message = self._format_result(stats)
+        return 'result'
+
+    @classmethod
+    def _import_rows(
+            cls, rows, create_missing_price_index=False,
+            overwrite_existing_price=False):
+        Price = Pool().get('price.price')
+        PriceValue = Pool().get('price.price_value')
+        stats = {
+            'created_indexes': 0,
+            'imported': 0,
+            'updated': 0,
+            'skipped': [],
+        }
+
+        for row_number, row in enumerate(rows, start=2):
+            price_index = (row.get('price_index') or '').strip()
+            price_date = row.get('price_date')
+            if not price_index:
+                stats['skipped'].append(
+                    cls._skip(row_number, '', 'missing price_index'))
+                continue
+            if not price_date:
+                stats['skipped'].append(
+                    cls._skip(row_number, price_index, 'missing price_date'))
+                continue
+
+            prices = Price.search([('price_index', '=', price_index)], limit=1)
+            if prices:
+                price = prices[0]
+            elif create_missing_price_index:
+                price, = Price.create([{
+                    'price_index': price_index,
+                    'price_desc': price_index,
+                }])
+                stats['created_indexes'] += 1
+            else:
+                stats['skipped'].append(
+                    cls._skip(row_number, price_index, 'price_index missing'))
+                continue
+
+            values = cls._price_value_values(price, row)
+            existing = PriceValue.search([
+                ('price', '=', price.id),
+                ('price_date', '=', price_date),
+            ], limit=1)
+
+            if existing:
+                if overwrite_existing_price:
+                    PriceValue.write(existing, values)
+                    stats['updated'] += 1
+                else:
+                    stats['skipped'].append(cls._skip(
+                        row_number, price_index,
+                        'price_date already exists'))
+                continue
+
+            PriceValue.create([values])
+            stats['imported'] += 1
+
+        return stats
+
+    @classmethod
+    def _price_value_values(cls, price, row):
+        return {
+            'price': price.id,
+            'price_date': row['price_date'],
+            'high_price': cls._as_float(row.get('high_price')),
+            'low_price': cls._as_float(row.get('low_price')),
+            'open_price': cls._as_float(row.get('open_price')),
+            'price_value': cls._as_float(row.get('price_value')),
+        }
+
+    @staticmethod
+    def _skip(row_number, price_index, reason):
+        return {
+            'row': row_number,
+            'price_index': price_index,
+            'reason': reason,
+        }
+
+    @classmethod
+    def _format_result(cls, stats):
+        lines = [
+            'Import completed.',
+            f"Created price indexes: {stats['created_indexes']}",
+            f"Imported prices: {stats['imported']}",
+            f"Updated prices: {stats['updated']}",
+            f"Skipped rows: {len(stats['skipped'])}",
+        ]
+        if stats['skipped']:
+            lines.append('')
+            lines.append('Skipped details:')
+            for skip in stats['skipped']:
+                label = skip['price_index'] or '<empty>'
+                lines.append(
+                    f"Row {skip['row']} - {label}: {skip['reason']}")
+        return '\n'.join(lines)
+
+    @classmethod
+    def _read_xlsx(cls, data):
+        try:
+            with zipfile.ZipFile(BytesIO(data)) as workbook:
+                shared_strings = cls._read_shared_strings(workbook)
+                sheet_name = cls._first_sheet_name(workbook)
+                sheet = ElementTree.fromstring(workbook.read(sheet_name))
+        except (KeyError, zipfile.BadZipFile, ElementTree.ParseError):
+            raise UserError("The selected file is not a valid Excel .xlsx file.")
+
+        ns = {'s': 'http://schemas.openxmlformats.org/spreadsheetml/2006/main'}
+        rows = sheet.findall('.//s:sheetData/s:row', ns)
+        if not rows:
+            return []
+
+        headers = {}
+        for cell in rows[0].findall('s:c', ns):
+            index = cls._cell_column_index(cell.get('r'))
+            header = cls._cell_value(cell, shared_strings)
+            normalized = cls._normalize_header(header)
+            if normalized in cls.REQUIRED_COLUMNS:
+                headers[index] = cls.REQUIRED_COLUMNS[normalized]
+
+        missing = set(cls.REQUIRED_COLUMNS.values()) - set(headers.values())
+        if missing:
+            raise UserError(
+                "Missing columns in Excel file: %s"
+                % ', '.join(sorted(missing)))
+
+        result = []
+        for sheet_row in rows[1:]:
+            values = {}
+            for cell in sheet_row.findall('s:c', ns):
+                index = cls._cell_column_index(cell.get('r'))
+                field = headers.get(index)
+                if field:
+                    values[field] = cls._cell_value(cell, shared_strings)
+            if any(v not in (None, '') for v in values.values()):
+                values['price_date'] = cls._as_date(values.get('price_date'))
+                result.append(values)
+        return result
+
+    @staticmethod
+    def _read_shared_strings(workbook):
+        try:
+            content = workbook.read('xl/sharedStrings.xml')
+        except KeyError:
+            return []
+        root = ElementTree.fromstring(content)
+        ns = {'s': 'http://schemas.openxmlformats.org/spreadsheetml/2006/main'}
+        strings = []
+        for item in root.findall('s:si', ns):
+            strings.append(''.join(
+                text.text or '' for text in item.findall('.//s:t', ns)))
+        return strings
+
+    @staticmethod
+    def _first_sheet_name(workbook):
+        workbook_xml = ElementTree.fromstring(workbook.read('xl/workbook.xml'))
+        rels_xml = ElementTree.fromstring(
+            workbook.read('xl/_rels/workbook.xml.rels'))
+        wb_ns = {
+            's': 'http://schemas.openxmlformats.org/spreadsheetml/2006/main',
+            'r': (
+                'http://schemas.openxmlformats.org/officeDocument/2006/'
+                'relationships'),
+        }
+        rel_ns = {
+            'r': (
+                'http://schemas.openxmlformats.org/package/2006/'
+                'relationships'),
+        }
+        sheet = workbook_xml.find('.//s:sheet', wb_ns)
+        rel_id = sheet.get(
+            '{http://schemas.openxmlformats.org/officeDocument/2006/'
+            'relationships}id')
+        for rel in rels_xml.findall('r:Relationship', rel_ns):
+            if rel.get('Id') == rel_id:
+                target = rel.get('Target')
+                if not target.startswith('/'):
+                    target = 'xl/' + target
+                return target.lstrip('/')
+        raise KeyError('No worksheet found')
+
+    @staticmethod
+    def _cell_column_index(reference):
+        match = re.match(r'([A-Z]+)', reference or '')
+        if not match:
+            return None
+        index = 0
+        for char in match.group(1):
+            index = index * 26 + ord(char) - ord('A') + 1
+        return index
+
+    @classmethod
+    def _cell_value(cls, cell, shared_strings):
+        ns = {'s': 'http://schemas.openxmlformats.org/spreadsheetml/2006/main'}
+        if cell.get('t') == 'inlineStr':
+            text = cell.find('.//s:t', ns)
+            return text.text if text is not None else ''
+        value = cell.find('s:v', ns)
+        if value is None:
+            return ''
+        if cell.get('t') == 's':
+            return shared_strings[int(value.text)]
+        return value.text
+
+    @staticmethod
+    def _normalize_header(value):
+        return re.sub(r'[^a-z0-9]', '', (value or '').strip().lower())
+
+    @staticmethod
+    def _as_float(value):
+        if value in (None, ''):
+            return None
+        return float(str(value).replace(',', '.'))
+
+    @staticmethod
+    def _as_date(value):
+        if isinstance(value, datetime.date):
+            return value
+        if value in (None, ''):
+            return None
+        text = str(value).strip()
+        if re.match(r'^\d+(\.\d+)?$', text):
+            return (
+                datetime.date(1899, 12, 30)
+                + datetime.timedelta(days=int(float(text))))
+        for fmt in ('%Y-%m-%d', '%d/%m/%Y', '%m/%d/%Y'):
+            try:
+                return datetime.datetime.strptime(text, fmt).date()
+            except ValueError:
+                pass
+        raise UserError("Invalid price_date: %s" % text)
 
 class MtmScenario(ModelSQL, ModelView):
     "MtM Scenario"

modules/purchase_trade/pricing.xml

@@ -20,12 +20,29 @@ this repository contains the full copyright notices and license terms. -->
             <field name="priority" eval="20"/>
             <field name="name">summary_tree_sequence</field>
         </record>
-        <record model="ir.ui.view" id="estimated_view_tree">
-            <field name="model">pricing.estimated</field>
-            <field name="type">tree</field>
-            <field name="priority" eval="10"/>
-            <field name="name">estimated_tree</field>
-        </record>
+        <record model="ir.ui.view" id="estimated_view_tree">
+            <field name="model">pricing.estimated</field>
+            <field name="type">tree</field>
+            <field name="priority" eval="10"/>
+            <field name="name">estimated_tree</field>
+        </record>
+
+        <record model="ir.ui.view" id="import_prices_start_view_form">
+            <field name="model">purchase_trade.import_prices.start</field>
+            <field name="type">form</field>
+            <field name="name">import_prices_start_form</field>
+        </record>
+        <record model="ir.ui.view" id="import_prices_result_view_form">
+            <field name="model">purchase_trade.import_prices.result</field>
+            <field name="type">form</field>
+            <field name="name">import_prices_result_form</field>
+        </record>
+
+        <record model="ir.action.wizard" id="act_import_prices">
+            <field name="name">Import Prices</field>
+            <field name="wiz_name">purchase_trade.import_prices</field>
+            <field name="window" eval="True"/>
+        </record>
 
         <record model="ir.ui.view" id="component_view_tree">
             <field name="model">pricing.component</field>
@@ -181,12 +198,17 @@ this repository contains the full copyright notices and license terms. -->
             sequence="99"
             id="menu_mtm"
             icon="tradon-mtm" />
-        <menuitem
-            name="Strategy"
-            action="act_strategy_form"
-            parent="menu_mtm"
-            sequence="10"
-            id="menu_strategy" />
-        
-    </data>
-</tryton>
+        <menuitem
+            name="Strategy"
+            action="act_strategy_form"
+            parent="menu_mtm"
+            sequence="10"
+            id="menu_strategy" />
+        <menuitem
+            parent="price.menu_price"
+            sequence="85"
+            action="act_import_prices"
+            id="menu_import_prices"/>
+        
+    </data>
+</tryton>

modules/purchase_trade/tests/test_module.py

@@ -2,10 +2,13 @@
 # this repository contains the full copyright notices and license terms.
 
 from decimal import Decimal
+from datetime import date
+from types import SimpleNamespace
 from unittest.mock import Mock, patch
 
 from trytond.pool import Pool
 from trytond.tests.test_tryton import ModuleTestCase, with_transaction
+from trytond.modules.purchase_trade.pricing import ImportPrices
 
 
 class PurchaseTradeTestCase(ModuleTestCase):
@@ -70,5 +73,152 @@ class PurchaseTradeTestCase(ModuleTestCase):
             strategy.get_mtm(line, Decimal('10')),
             Decimal('250.00'))
 
+    def test_import_prices_skips_missing_index_without_create_option(self):
+        'import prices skips rows when the price index is missing'
+        price_model = _FakePriceModel()
+        price_value_model = _FakePriceValueModel()
+
+        with patch('trytond.modules.purchase_trade.pricing.Pool') as pool:
+            pool.return_value.get.side_effect = {
+                'price.price': price_model,
+                'price.price_value': price_value_model,
+            }.__getitem__
+
+            stats = ImportPrices._import_rows([{
+                'price_index': 'LME Copper',
+                'price_date': date(2026, 3, 27),
+                'high_price': '100',
+                'low_price': '90',
+                'open_price': '95',
+                'price_value': '98',
+            }])
+
+        self.assertEqual(stats['imported'], 0)
+        self.assertEqual(len(stats['skipped']), 1)
+        self.assertEqual(stats['skipped'][0]['reason'], 'price_index missing')
+
+    def test_import_prices_creates_index_and_imports_new_price(self):
+        'import prices creates missing index when requested'
+        price_model = _FakePriceModel()
+        price_value_model = _FakePriceValueModel()
+
+        with patch('trytond.modules.purchase_trade.pricing.Pool') as pool:
+            pool.return_value.get.side_effect = {
+                'price.price': price_model,
+                'price.price_value': price_value_model,
+            }.__getitem__
+
+            stats = ImportPrices._import_rows([{
+                'price_index': 'LME Copper',
+                'price_date': date(2026, 3, 27),
+                'high_price': '100',
+                'low_price': '90',
+                'open_price': '95',
+                'price_value': '98',
+            }], create_missing_price_index=True)
+
+        self.assertEqual(stats['created_indexes'], 1)
+        self.assertEqual(stats['imported'], 1)
+        self.assertEqual(price_model.records[0].price_index, 'LME Copper')
+        self.assertEqual(price_value_model.records[0].price_value, 98.0)
+
+    def test_import_prices_skips_existing_date_without_overwrite(self):
+        'import prices skips existing price dates unless overwrite is enabled'
+        price = SimpleNamespace(id=1, price_index='LME Copper')
+        price_model = _FakePriceModel([price])
+        price_value_model = _FakePriceValueModel([
+            SimpleNamespace(
+                id=1, price=1, price_date=date(2026, 3, 27),
+                price_value=97.0)])
+
+        with patch('trytond.modules.purchase_trade.pricing.Pool') as pool:
+            pool.return_value.get.side_effect = {
+                'price.price': price_model,
+                'price.price_value': price_value_model,
+            }.__getitem__
+
+            stats = ImportPrices._import_rows([{
+                'price_index': 'LME Copper',
+                'price_date': date(2026, 3, 27),
+                'price_value': '98',
+            }])
+
+        self.assertEqual(stats['updated'], 0)
+        self.assertEqual(price_value_model.records[0].price_value, 97.0)
+        self.assertEqual(
+            stats['skipped'][0]['reason'], 'price_date already exists')
+
+    def test_import_prices_overwrites_existing_date_when_requested(self):
+        'import prices updates existing price dates when requested'
+        price = SimpleNamespace(id=1, price_index='LME Copper')
+        price_model = _FakePriceModel([price])
+        price_value_model = _FakePriceValueModel([
+            SimpleNamespace(
+                id=1, price=1, price_date=date(2026, 3, 27),
+                price_value=97.0)])
+
+        with patch('trytond.modules.purchase_trade.pricing.Pool') as pool:
+            pool.return_value.get.side_effect = {
+                'price.price': price_model,
+                'price.price_value': price_value_model,
+            }.__getitem__
+
+            stats = ImportPrices._import_rows([{
+                'price_index': 'LME Copper',
+                'price_date': date(2026, 3, 27),
+                'price_value': '98',
+            }], overwrite_existing_price=True)
+
+        self.assertEqual(stats['updated'], 1)
+        self.assertEqual(price_value_model.records[0].price_value, 98.0)
+
+
+class _FakePriceModel:
+
+    def __init__(self, records=None):
+        self.records = records or []
+
+    def search(self, domain, limit=None):
+        price_index = domain[0][2]
+        records = [
+            record for record in self.records
+            if record.price_index == price_index]
+        return records[:limit] if limit else records
+
+    def create(self, values):
+        records = []
+        for value in values:
+            record = SimpleNamespace(id=len(self.records) + 1, **value)
+            self.records.append(record)
+            records.append(record)
+        return records
+
+
+class _FakePriceValueModel:
+
+    def __init__(self, records=None):
+        self.records = records or []
+
+    def search(self, domain, limit=None):
+        price = domain[0][2]
+        price_date = domain[1][2]
+        records = [
+            record for record in self.records
+            if record.price == price and record.price_date == price_date]
+        return records[:limit] if limit else records
+
+    def create(self, values):
+        records = []
+        for value in values:
+            record = SimpleNamespace(id=len(self.records) + 1, **value)
+            self.records.append(record)
+            records.append(record)
+        return records
+
+    def write(self, records, values):
+        for record in records:
+            for name, value in values.items():
+                setattr(record, name, value)
+
 
 del ModuleTestCase

modules/purchase_trade/view/import_prices_result_form.xml

@@ -0,0 +1,3 @@
+<form>
+    <field name="message"/>
+</form>

modules/purchase_trade/view/import_prices_start_form.xml

@@ -0,0 +1,10 @@
+<form col="4">
+    <label name="file_"/>
+    <field name="file_"/>
+    <newline/>
+    <label name="create_missing_price_index"/>
+    <field name="create_missing_price_index"/>
+    <newline/>
+    <label name="overwrite_existing_price"/>
+    <field name="overwrite_existing_price"/>
+</form>