diff --git a/modules/purchase_trade/__init__.py b/modules/purchase_trade/__init__.py index d3bae59..f846458 100755 --- a/modules/purchase_trade/__init__.py +++ b/modules/purchase_trade/__init__.py @@ -55,15 +55,18 @@ def register(): lc.LCMT700, lc.LCMessage, lc.CreateLCStart, - global_reporting.GRConfiguration, - module='purchase_trade', type_='model') + global_reporting.GRConfiguration, + pricing.ImportPricesStart, + pricing.ImportPricesResult, + module='purchase_trade', type_='model') Pool.register( - incoming.ImportSwift, - incoming.PrepareDocuments, - incoming.AnalyzeConditions, - lc.CreateLCWizard, - module='purchase_trade', type_='wizard' - ) + incoming.ImportSwift, + incoming.PrepareDocuments, + incoming.AnalyzeConditions, + lc.CreateLCWizard, + pricing.ImportPrices, + module='purchase_trade', type_='wizard' + ) Pool.register( credit_risk.Party, credit_risk.CreditRiskRule, diff --git a/modules/purchase_trade/docs/business-rules.md b/modules/purchase_trade/docs/business-rules.md index 3095758..8763545 100644 --- a/modules/purchase_trade/docs/business-rules.md +++ b/modules/purchase_trade/docs/business-rules.md @@ -155,3 +155,21 @@ Pour cette regle, couvrir au minimum: - augmentation sans `lot.qt` ouvert - diminution possible - diminution impossible avec erreur + + +### BR-PT-004 - Market Price Import + +- Goal: import market prices from an Excel file +- Description: after having selected an Excel file with the following columns: + - price_index: name of the market + - price_date: date of the price + - high price: high price + - low price: low price + - open price: open price + - price value: price value +- Expected behavior: + - Check if price_index exists prior to impport related prices + - If price_index does not exist, check if option "Create price index is missing" has been checked in the input screen. If checked then try to create first the price index (table: price_price). If option is not checked, then ignore the price_index and mark it as "skipped" in the output results of the import. + - Check if price_date for the price_index already exists. + - If price_date does not exist, then import the price. If price_date already exists then check if option "Overwrite existing price" is checked in the input screen. If checked, update the price for price_index and price_date. If option is not checked, then ignore the price_date and mark as "skipped" in the output. + diff --git a/modules/purchase_trade/pricing.py b/modules/purchase_trade/pricing.py index 6191627..6c9f74e 100755 --- a/modules/purchase_trade/pricing.py +++ b/modules/purchase_trade/pricing.py @@ -1,9 +1,10 @@ # This file is part of Tryton. The COPYRIGHT file at the top level of # this repository contains the full copyright notices and license terms. -from trytond.model import fields -from trytond.pool import Pool, PoolMeta -from trytond.pyson import Bool, Eval, Id -from trytond.model import (ModelSQL, ModelView) +from trytond.model import fields +from trytond.exceptions import UserError +from trytond.pool import Pool, PoolMeta +from trytond.pyson import Bool, Eval, Id +from trytond.model import (ModelSQL, ModelView) from trytond.tools import is_full_text, lstrip_wildcard from trytond.transaction import Transaction, inactive_records from decimal import getcontext, Decimal, ROUND_HALF_UP @@ -11,12 +12,16 @@ from sql.aggregate import Count, Max, Min, Sum, Avg, BoolOr from sql.conditionals import Case from sql import Column, Literal from sql.functions import CurrentTimestamp, DateTrunc -from trytond.wizard import Button, StateTransition, StateView, Wizard -from itertools import chain, groupby -from operator import itemgetter -import datetime -import logging -from trytond.modules.purchase_trade.purchase import (TRIGGERS) +from trytond.wizard import Button, StateTransition, StateView, Wizard +from itertools import chain, groupby +from operator import itemgetter +import datetime +import logging +import re +import zipfile +from io import BytesIO +from xml.etree import ElementTree +from trytond.modules.purchase_trade.purchase import (TRIGGERS) logger = logging.getLogger(__name__) @@ -50,13 +55,302 @@ DAYS = [ ('sunday', 'Sunday'), ] -class Estimated(ModelSQL, ModelView): - "Estimated date" - __name__ = 'pricing.estimated' +class Estimated(ModelSQL, ModelView): + "Estimated date" + __name__ = 'pricing.estimated' trigger = fields.Selection(TRIGGERS,"Trigger") - estimated_date = fields.Date("Estimated date") - fin_int_delta = fields.Integer("Financing interests delta") + estimated_date = fields.Date("Estimated date") + fin_int_delta = fields.Integer("Financing interests delta") + + +class ImportPricesStart(ModelView): + "Import Prices" + __name__ = 'purchase_trade.import_prices.start' + + file_ = fields.Binary('Excel file', required=True, filename='filename') + filename = fields.Char('Filename') + create_missing_price_index = fields.Boolean( + "Create price index if missing") + overwrite_existing_price = fields.Boolean("Overwrite existing price") + + +class ImportPricesResult(ModelView): + "Import Prices Result" + __name__ = 'purchase_trade.import_prices.result' + + message = fields.Text("Results", readonly=True) + + +class ImportPrices(Wizard): + "Import Prices" + __name__ = 'purchase_trade.import_prices' + + start = StateView( + 'purchase_trade.import_prices.start', + 'purchase_trade.import_prices_start_view_form', + [ + Button('Cancel', 'end', 'tryton-cancel'), + Button('Import', 'import_', 'tryton-ok', default=True), + ]) + import_ = StateTransition() + result = StateView( + 'purchase_trade.import_prices.result', + 'purchase_trade.import_prices_result_view_form', + [ + Button('OK', 'end', 'tryton-ok', default=True), + ]) + + REQUIRED_COLUMNS = { + 'priceindex': 'price_index', + 'pricedate': 'price_date', + 'highprice': 'high_price', + 'lowprice': 'low_price', + 'openprice': 'open_price', + 'pricevalue': 'price_value', + } + + def transition_import_(self): + rows = self._read_xlsx(self.start.file_) + stats = self._import_rows( + rows, + create_missing_price_index=( + self.start.create_missing_price_index), + overwrite_existing_price=self.start.overwrite_existing_price) + self.result.message = self._format_result(stats) + return 'result' + + @classmethod + def _import_rows( + cls, rows, create_missing_price_index=False, + overwrite_existing_price=False): + Price = Pool().get('price.price') + PriceValue = Pool().get('price.price_value') + stats = { + 'created_indexes': 0, + 'imported': 0, + 'updated': 0, + 'skipped': [], + } + + for row_number, row in enumerate(rows, start=2): + price_index = (row.get('price_index') or '').strip() + price_date = row.get('price_date') + if not price_index: + stats['skipped'].append( + cls._skip(row_number, '', 'missing price_index')) + continue + if not price_date: + stats['skipped'].append( + cls._skip(row_number, price_index, 'missing price_date')) + continue + + prices = Price.search([('price_index', '=', price_index)], limit=1) + if prices: + price = prices[0] + elif create_missing_price_index: + price, = Price.create([{ + 'price_index': price_index, + 'price_desc': price_index, + }]) + stats['created_indexes'] += 1 + else: + stats['skipped'].append( + cls._skip(row_number, price_index, 'price_index missing')) + continue + + values = cls._price_value_values(price, row) + existing = PriceValue.search([ + ('price', '=', price.id), + ('price_date', '=', price_date), + ], limit=1) + + if existing: + if overwrite_existing_price: + PriceValue.write(existing, values) + stats['updated'] += 1 + else: + stats['skipped'].append(cls._skip( + row_number, price_index, + 'price_date already exists')) + continue + + PriceValue.create([values]) + stats['imported'] += 1 + + return stats + + @classmethod + def _price_value_values(cls, price, row): + return { + 'price': price.id, + 'price_date': row['price_date'], + 'high_price': cls._as_float(row.get('high_price')), + 'low_price': cls._as_float(row.get('low_price')), + 'open_price': cls._as_float(row.get('open_price')), + 'price_value': cls._as_float(row.get('price_value')), + } + + @staticmethod + def _skip(row_number, price_index, reason): + return { + 'row': row_number, + 'price_index': price_index, + 'reason': reason, + } + + @classmethod + def _format_result(cls, stats): + lines = [ + 'Import completed.', + f"Created price indexes: {stats['created_indexes']}", + f"Imported prices: {stats['imported']}", + f"Updated prices: {stats['updated']}", + f"Skipped rows: {len(stats['skipped'])}", + ] + if stats['skipped']: + lines.append('') + lines.append('Skipped details:') + for skip in stats['skipped']: + label = skip['price_index'] or '' + lines.append( + f"Row {skip['row']} - {label}: {skip['reason']}") + return '\n'.join(lines) + + @classmethod + def _read_xlsx(cls, data): + try: + with zipfile.ZipFile(BytesIO(data)) as workbook: + shared_strings = cls._read_shared_strings(workbook) + sheet_name = cls._first_sheet_name(workbook) + sheet = ElementTree.fromstring(workbook.read(sheet_name)) + except (KeyError, zipfile.BadZipFile, ElementTree.ParseError): + raise UserError("The selected file is not a valid Excel .xlsx file.") + + ns = {'s': 'http://schemas.openxmlformats.org/spreadsheetml/2006/main'} + rows = sheet.findall('.//s:sheetData/s:row', ns) + if not rows: + return [] + + headers = {} + for cell in rows[0].findall('s:c', ns): + index = cls._cell_column_index(cell.get('r')) + header = cls._cell_value(cell, shared_strings) + normalized = cls._normalize_header(header) + if normalized in cls.REQUIRED_COLUMNS: + headers[index] = cls.REQUIRED_COLUMNS[normalized] + + missing = set(cls.REQUIRED_COLUMNS.values()) - set(headers.values()) + if missing: + raise UserError( + "Missing columns in Excel file: %s" + % ', '.join(sorted(missing))) + + result = [] + for sheet_row in rows[1:]: + values = {} + for cell in sheet_row.findall('s:c', ns): + index = cls._cell_column_index(cell.get('r')) + field = headers.get(index) + if field: + values[field] = cls._cell_value(cell, shared_strings) + if any(v not in (None, '') for v in values.values()): + values['price_date'] = cls._as_date(values.get('price_date')) + result.append(values) + return result + + @staticmethod + def _read_shared_strings(workbook): + try: + content = workbook.read('xl/sharedStrings.xml') + except KeyError: + return [] + root = ElementTree.fromstring(content) + ns = {'s': 'http://schemas.openxmlformats.org/spreadsheetml/2006/main'} + strings = [] + for item in root.findall('s:si', ns): + strings.append(''.join( + text.text or '' for text in item.findall('.//s:t', ns))) + return strings + + @staticmethod + def _first_sheet_name(workbook): + workbook_xml = ElementTree.fromstring(workbook.read('xl/workbook.xml')) + rels_xml = ElementTree.fromstring( + workbook.read('xl/_rels/workbook.xml.rels')) + wb_ns = { + 's': 'http://schemas.openxmlformats.org/spreadsheetml/2006/main', + 'r': ( + 'http://schemas.openxmlformats.org/officeDocument/2006/' + 'relationships'), + } + rel_ns = { + 'r': ( + 'http://schemas.openxmlformats.org/package/2006/' + 'relationships'), + } + sheet = workbook_xml.find('.//s:sheet', wb_ns) + rel_id = sheet.get( + '{http://schemas.openxmlformats.org/officeDocument/2006/' + 'relationships}id') + for rel in rels_xml.findall('r:Relationship', rel_ns): + if rel.get('Id') == rel_id: + target = rel.get('Target') + if not target.startswith('/'): + target = 'xl/' + target + return target.lstrip('/') + raise KeyError('No worksheet found') + + @staticmethod + def _cell_column_index(reference): + match = re.match(r'([A-Z]+)', reference or '') + if not match: + return None + index = 0 + for char in match.group(1): + index = index * 26 + ord(char) - ord('A') + 1 + return index + + @classmethod + def _cell_value(cls, cell, shared_strings): + ns = {'s': 'http://schemas.openxmlformats.org/spreadsheetml/2006/main'} + if cell.get('t') == 'inlineStr': + text = cell.find('.//s:t', ns) + return text.text if text is not None else '' + value = cell.find('s:v', ns) + if value is None: + return '' + if cell.get('t') == 's': + return shared_strings[int(value.text)] + return value.text + + @staticmethod + def _normalize_header(value): + return re.sub(r'[^a-z0-9]', '', (value or '').strip().lower()) + + @staticmethod + def _as_float(value): + if value in (None, ''): + return None + return float(str(value).replace(',', '.')) + + @staticmethod + def _as_date(value): + if isinstance(value, datetime.date): + return value + if value in (None, ''): + return None + text = str(value).strip() + if re.match(r'^\d+(\.\d+)?$', text): + return ( + datetime.date(1899, 12, 30) + + datetime.timedelta(days=int(float(text)))) + for fmt in ('%Y-%m-%d', '%d/%m/%Y', '%m/%d/%Y'): + try: + return datetime.datetime.strptime(text, fmt).date() + except ValueError: + pass + raise UserError("Invalid price_date: %s" % text) class MtmScenario(ModelSQL, ModelView): "MtM Scenario" diff --git a/modules/purchase_trade/pricing.xml b/modules/purchase_trade/pricing.xml index 5332ab0..2d3d6a9 100755 --- a/modules/purchase_trade/pricing.xml +++ b/modules/purchase_trade/pricing.xml @@ -20,12 +20,29 @@ this repository contains the full copyright notices and license terms. --> summary_tree_sequence - - pricing.estimated - tree - - estimated_tree - + + pricing.estimated + tree + + estimated_tree + + + + purchase_trade.import_prices.start + form + import_prices_start_form + + + purchase_trade.import_prices.result + form + import_prices_result_form + + + + Import Prices + purchase_trade.import_prices + + pricing.component @@ -181,12 +198,17 @@ this repository contains the full copyright notices and license terms. --> sequence="99" id="menu_mtm" icon="tradon-mtm" /> - - - - \ No newline at end of file + + + + + diff --git a/modules/purchase_trade/tests/test_module.py b/modules/purchase_trade/tests/test_module.py index 7d2b4c5..1592ece 100644 --- a/modules/purchase_trade/tests/test_module.py +++ b/modules/purchase_trade/tests/test_module.py @@ -2,10 +2,13 @@ # this repository contains the full copyright notices and license terms. from decimal import Decimal +from datetime import date +from types import SimpleNamespace from unittest.mock import Mock, patch from trytond.pool import Pool from trytond.tests.test_tryton import ModuleTestCase, with_transaction +from trytond.modules.purchase_trade.pricing import ImportPrices class PurchaseTradeTestCase(ModuleTestCase): @@ -70,5 +73,152 @@ class PurchaseTradeTestCase(ModuleTestCase): strategy.get_mtm(line, Decimal('10')), Decimal('250.00')) + def test_import_prices_skips_missing_index_without_create_option(self): + 'import prices skips rows when the price index is missing' + price_model = _FakePriceModel() + price_value_model = _FakePriceValueModel() + + with patch('trytond.modules.purchase_trade.pricing.Pool') as pool: + pool.return_value.get.side_effect = { + 'price.price': price_model, + 'price.price_value': price_value_model, + }.__getitem__ + + stats = ImportPrices._import_rows([{ + 'price_index': 'LME Copper', + 'price_date': date(2026, 3, 27), + 'high_price': '100', + 'low_price': '90', + 'open_price': '95', + 'price_value': '98', + }]) + + self.assertEqual(stats['imported'], 0) + self.assertEqual(len(stats['skipped']), 1) + self.assertEqual(stats['skipped'][0]['reason'], 'price_index missing') + + def test_import_prices_creates_index_and_imports_new_price(self): + 'import prices creates missing index when requested' + price_model = _FakePriceModel() + price_value_model = _FakePriceValueModel() + + with patch('trytond.modules.purchase_trade.pricing.Pool') as pool: + pool.return_value.get.side_effect = { + 'price.price': price_model, + 'price.price_value': price_value_model, + }.__getitem__ + + stats = ImportPrices._import_rows([{ + 'price_index': 'LME Copper', + 'price_date': date(2026, 3, 27), + 'high_price': '100', + 'low_price': '90', + 'open_price': '95', + 'price_value': '98', + }], create_missing_price_index=True) + + self.assertEqual(stats['created_indexes'], 1) + self.assertEqual(stats['imported'], 1) + self.assertEqual(price_model.records[0].price_index, 'LME Copper') + self.assertEqual(price_value_model.records[0].price_value, 98.0) + + def test_import_prices_skips_existing_date_without_overwrite(self): + 'import prices skips existing price dates unless overwrite is enabled' + price = SimpleNamespace(id=1, price_index='LME Copper') + price_model = _FakePriceModel([price]) + price_value_model = _FakePriceValueModel([ + SimpleNamespace( + id=1, price=1, price_date=date(2026, 3, 27), + price_value=97.0)]) + + with patch('trytond.modules.purchase_trade.pricing.Pool') as pool: + pool.return_value.get.side_effect = { + 'price.price': price_model, + 'price.price_value': price_value_model, + }.__getitem__ + + stats = ImportPrices._import_rows([{ + 'price_index': 'LME Copper', + 'price_date': date(2026, 3, 27), + 'price_value': '98', + }]) + + self.assertEqual(stats['updated'], 0) + self.assertEqual(price_value_model.records[0].price_value, 97.0) + self.assertEqual( + stats['skipped'][0]['reason'], 'price_date already exists') + + def test_import_prices_overwrites_existing_date_when_requested(self): + 'import prices updates existing price dates when requested' + price = SimpleNamespace(id=1, price_index='LME Copper') + price_model = _FakePriceModel([price]) + price_value_model = _FakePriceValueModel([ + SimpleNamespace( + id=1, price=1, price_date=date(2026, 3, 27), + price_value=97.0)]) + + with patch('trytond.modules.purchase_trade.pricing.Pool') as pool: + pool.return_value.get.side_effect = { + 'price.price': price_model, + 'price.price_value': price_value_model, + }.__getitem__ + + stats = ImportPrices._import_rows([{ + 'price_index': 'LME Copper', + 'price_date': date(2026, 3, 27), + 'price_value': '98', + }], overwrite_existing_price=True) + + self.assertEqual(stats['updated'], 1) + self.assertEqual(price_value_model.records[0].price_value, 98.0) + + +class _FakePriceModel: + + def __init__(self, records=None): + self.records = records or [] + + def search(self, domain, limit=None): + price_index = domain[0][2] + records = [ + record for record in self.records + if record.price_index == price_index] + return records[:limit] if limit else records + + def create(self, values): + records = [] + for value in values: + record = SimpleNamespace(id=len(self.records) + 1, **value) + self.records.append(record) + records.append(record) + return records + + +class _FakePriceValueModel: + + def __init__(self, records=None): + self.records = records or [] + + def search(self, domain, limit=None): + price = domain[0][2] + price_date = domain[1][2] + records = [ + record for record in self.records + if record.price == price and record.price_date == price_date] + return records[:limit] if limit else records + + def create(self, values): + records = [] + for value in values: + record = SimpleNamespace(id=len(self.records) + 1, **value) + self.records.append(record) + records.append(record) + return records + + def write(self, records, values): + for record in records: + for name, value in values.items(): + setattr(record, name, value) + del ModuleTestCase diff --git a/modules/purchase_trade/view/import_prices_result_form.xml b/modules/purchase_trade/view/import_prices_result_form.xml new file mode 100644 index 0000000..dd43be2 --- /dev/null +++ b/modules/purchase_trade/view/import_prices_result_form.xml @@ -0,0 +1,3 @@ +
+ + diff --git a/modules/purchase_trade/view/import_prices_start_form.xml b/modules/purchase_trade/view/import_prices_start_form.xml new file mode 100644 index 0000000..399f7f3 --- /dev/null +++ b/modules/purchase_trade/view/import_prices_start_form.xml @@ -0,0 +1,10 @@ +
+