diff --git a/modules/purchase_trade/pricing.py b/modules/purchase_trade/pricing.py index 0dd6f5f..e39a780 100755 --- a/modules/purchase_trade/pricing.py +++ b/modules/purchase_trade/pricing.py @@ -83,10 +83,11 @@ class ImportPricesStart(ModelView): "Import Prices" __name__ = 'purchase_trade.import_prices.start' - file_structure = fields.Selection([ - ('historical', "Historical Prices"), - ('forward', "Forward Prices"), - ], "Excel file structure", required=True) + file_structure = fields.Selection([ + ('historical', "Historical Prices"), + ('historical_all_sources', "Historical Prices (All Sources)"), + ('forward', "Forward Prices"), + ], "Excel file structure", required=True) file_ = fields.Binary('Excel file', required=True, filename='filename') filename = fields.Char('Filename') create_missing_price_index = fields.Boolean( @@ -123,15 +124,20 @@ class ImportPrices(Wizard): Button('OK', 'end', 'tryton-ok', default=True), ]) - REQUIRED_COLUMNS = { - 'priceindex': 'price_index', - 'pricedate': 'price_date', + REQUIRED_COLUMNS = { + 'priceindex': 'price_index', + 'pricedate': 'price_date', 'highprice': 'high_price', 'lowprice': 'low_price', 'midprice': 'mid_price', 'openprice': 'open_price', 'pricevalue': 'price_value', - } + } + PRICE_INDEX_MAPPING_COLUMNS = ( + 'excel_tab', 'original_price_index', 'source', 'column_destination', + 'standard_price_index', 'index_description') + ALL_SOURCES_VALUE_COLUMNS = { + 'price_value', 'open_price', 'low_price', 'mid_price', 'high_price'} def transition_import_(self): rows = self._read_xlsx( @@ -182,16 +188,16 @@ class ImportPrices(Wizard): 'missing price_date')) continue - prices = Price.search( - [('price_index', '=', price_index)], limit=1) - if prices: - price = prices[0] - elif create_missing_price_index: - price, = Price.create([ - cls._price_index_values(price_index)]) - stats['created_indexes'].append( - cls._result_line( - row_number, price_index, None, + prices = Price.search( + [('price_index', '=', price_index)], limit=1) + if prices: + price = prices[0] + elif create_missing_price_index: + price, = Price.create([ + cls._price_index_values(price_index, row)]) + stats['created_indexes'].append( + cls._result_line( + row_number, price_index, None, 'price index created')) else: stats['skipped'].append( @@ -234,21 +240,26 @@ class ImportPrices(Wizard): return stats @classmethod - def _price_index_values(cls, price_index): - values = { - 'price_index': price_index, - 'price_desc': price_index, - 'price_curve_type': 'future', - } - references = [ - ('price_type', 'price.fixtype', [('name', '=', 'Market price')]), - ('price_currency', 'currency.currency', [('name', '=', 'USD')]), - ('price_calendar', 'price.calendar', [('name', '=', 'Argus EU')]), - ('price_unit', 'product.uom', [('name', '=', 'Mt')]), - ] - for field, model_name, domain in references: - record = cls._first_record(model_name, domain) - if record: + def _price_index_values(cls, price_index, row=None): + row = row or {} + values = { + 'price_index': price_index, + 'price_desc': row.get('_price_desc') or price_index, + 'price_curve_type': 'future', + } + references = [ + ('price_type', 'price.fixtype', [('name', '=', 'Market price')]), + ('price_currency', 'currency.currency', [('name', '=', 'USD')]), + ('price_calendar', 'price.calendar', [('name', '=', 'Argus EU')]), + ('price_unit', 'product.uom', [('name', '=', 'Mt')]), + ] + source = row.get('_source') + if source: + references.append( + ('price_area', 'price.area', [('name', '=', source)])) + for field, model_name, domain in references: + record = cls._first_record(model_name, domain) + if record: values[field] = record.id period = cls._period_from_price_index(price_index) @@ -368,13 +379,16 @@ class ImportPrices(Wizard): return ', '.join(parts) or 'price imported' @classmethod - def _read_xlsx(cls, data, file_structure='historical'): - try: - with zipfile.ZipFile(BytesIO(data)) as workbook: - shared_strings = cls._read_shared_strings(workbook) - sheet_name = cls._first_sheet_name(workbook) - sheet = ElementTree.fromstring(workbook.read(sheet_name)) - except (KeyError, zipfile.BadZipFile, ElementTree.ParseError): + def _read_xlsx(cls, data, file_structure='historical'): + try: + with zipfile.ZipFile(BytesIO(data)) as workbook: + shared_strings = cls._read_shared_strings(workbook) + if file_structure == 'historical_all_sources': + return cls._read_historical_all_sources_price_rows( + workbook, shared_strings) + sheet_name = cls._first_sheet_name(workbook) + sheet = ElementTree.fromstring(workbook.read(sheet_name)) + except (KeyError, zipfile.BadZipFile, ElementTree.ParseError): raise UserError("The selected file is not a valid Excel .xlsx file.") ns = {'s': 'http://schemas.openxmlformats.org/spreadsheetml/2006/main'} @@ -382,12 +396,169 @@ class ImportPrices(Wizard): if not rows: return [] - if file_structure == 'forward': - return cls._read_forward_price_rows(rows, shared_strings, ns) - return cls._read_historical_price_rows(rows, shared_strings, ns) - - @classmethod - def _read_historical_price_rows(cls, rows, shared_strings, ns): + if file_structure == 'forward': + return cls._read_forward_price_rows(rows, shared_strings, ns) + return cls._read_historical_price_rows(rows, shared_strings, ns) + + @classmethod + def _read_historical_all_sources_price_rows(cls, workbook, shared_strings): + ns = {'s': 'http://schemas.openxmlformats.org/spreadsheetml/2006/main'} + sheet_rows = cls._workbook_sheet_rows(workbook, ns) + mapping = cls._price_index_mapping() + return cls._read_historical_all_sources_rows( + sheet_rows, mapping, shared_strings, ns) + + @classmethod + def _read_historical_all_sources_rows( + cls, sheet_rows, mapping, shared_strings, ns): + result_by_key = {} + mapping_by_tab = {} + for mapping_row in mapping: + tab = mapping_row.get('excel_tab') + destination = mapping_row.get('column_destination') + if not tab or destination not in cls.ALL_SOURCES_VALUE_COLUMNS: + continue + mapping_by_tab.setdefault(tab, []).append(mapping_row) + + for tab, tab_mapping in mapping_by_tab.items(): + sheet_name = cls._matching_sheet_name(sheet_rows, tab) + if not sheet_name: + continue + rows = sheet_rows[sheet_name] + header_row_number, mapped_columns = cls._mapped_columns( + rows, tab_mapping, shared_strings, ns) + if not mapped_columns: + continue + date_column = cls._date_column( + rows[header_row_number - 1], mapped_columns, shared_strings, + ns) + + for excel_row_number, sheet_row in enumerate( + rows[header_row_number:], start=header_row_number + 1): + row_values = cls._row_values(sheet_row, shared_strings, ns) + price_date = row_values.get(date_column) + if price_date in (None, ''): + continue + for column, mapping_row in mapped_columns.items(): + price_value = row_values.get(column) + if price_value in (None, ''): + continue + price_index = ( + mapping_row.get('standard_price_index') or '').strip() + if not price_index: + continue + key = (price_index, price_date, excel_row_number) + result = result_by_key.setdefault(key, { + '_row_number': excel_row_number, + '_source': mapping_row.get('source'), + '_price_desc': mapping_row.get('index_description'), + 'price_index': price_index, + 'price_date': price_date, + }) + result[mapping_row['column_destination']] = price_value + if (mapping_row['column_destination'] == 'mid_price' + and not result.get('price_value')): + result['price_value'] = price_value + return list(result_by_key.values()) + + @classmethod + def _price_index_mapping(cls): + cursor = Transaction().connection.cursor() + columns = ', '.join(cls.PRICE_INDEX_MAPPING_COLUMNS) + cursor.execute( + 'SELECT %s FROM public."ext_H2SO4_PriceIndexMapping"' + % columns) + return [ + dict(zip(cls.PRICE_INDEX_MAPPING_COLUMNS, row)) + for row in cursor.fetchall()] + + @classmethod + def _mapped_columns(cls, rows, mapping, shared_strings, ns): + mapping_by_header = {} + for mapping_row in mapping: + header = cls._normalize_mapping_value( + mapping_row.get('original_price_index')) + mapping_by_header[header] = mapping_row + + hits_by_row = {} + columns_by_row = {} + for row_number, sheet_row in enumerate(rows, start=1): + for cell in sheet_row.findall('s:c', ns): + header = cls._normalize_mapping_value( + cls._cell_value(cell, shared_strings)) + mapping_row = mapping_by_header.get(header) + if not mapping_row: + continue + column = cls._cell_column_index(cell.get('r')) + hits_by_row[row_number] = hits_by_row.get(row_number, 0) + 1 + columns_by_row.setdefault(row_number, {})[column] = mapping_row + if not hits_by_row: + return None, {} + header_row_number = max(hits_by_row, key=hits_by_row.get) + return header_row_number, columns_by_row[header_row_number] + + @classmethod + def _date_column(cls, header_row, mapped_columns, shared_strings, ns): + for cell in header_row.findall('s:c', ns): + if cls._normalize_header( + cls._cell_value(cell, shared_strings)) == 'date': + return cls._cell_column_index(cell.get('r')) + return min(mapped_columns) - 1 + + @classmethod + def _row_values(cls, sheet_row, shared_strings, ns): + values = {} + for cell in sheet_row.findall('s:c', ns): + values[cls._cell_column_index(cell.get('r'))] = cls._cell_value( + cell, shared_strings) + return values + + @classmethod + def _workbook_sheet_rows(cls, workbook, ns): + sheet_rows = {} + for sheet_name, sheet_path in cls._worksheet_paths(workbook): + sheet = ElementTree.fromstring(workbook.read(sheet_path)) + sheet_rows[sheet_name] = sheet.findall('.//s:sheetData/s:row', ns) + return sheet_rows + + @classmethod + def _worksheet_paths(cls, workbook): + workbook_xml = ElementTree.fromstring(workbook.read('xl/workbook.xml')) + rels_xml = ElementTree.fromstring( + workbook.read('xl/_rels/workbook.xml.rels')) + wb_ns = { + 's': 'http://schemas.openxmlformats.org/spreadsheetml/2006/main', + 'r': ( + 'http://schemas.openxmlformats.org/officeDocument/2006/' + 'relationships'), + } + rel_ns = { + 'r': ( + 'http://schemas.openxmlformats.org/package/2006/' + 'relationships'), + } + rels = { + rel.get('Id'): rel.get('Target') + for rel in rels_xml.findall('r:Relationship', rel_ns)} + for sheet in workbook_xml.findall('.//s:sheet', wb_ns): + rel_id = sheet.get( + '{http://schemas.openxmlformats.org/officeDocument/2006/' + 'relationships}id') + target = rels[rel_id] + if not target.startswith('/'): + target = 'xl/' + target + yield sheet.get('name'), target.lstrip('/') + + @classmethod + def _matching_sheet_name(cls, sheet_rows, mapping_tab): + normalized_tab = cls._normalize_mapping_value(mapping_tab) + for sheet_name in sheet_rows: + if cls._normalize_mapping_value(sheet_name) == normalized_tab: + return sheet_name + return None + + @classmethod + def _read_historical_price_rows(cls, rows, shared_strings, ns): headers = {} for cell in rows[0].findall('s:c', ns): index = cls._cell_column_index(cell.get('r')) @@ -537,8 +708,12 @@ class ImportPrices(Wizard): return value.text @staticmethod - def _normalize_header(value): - return re.sub(r'[^a-z0-9]', '', (value or '').strip().lower()) + def _normalize_header(value): + return re.sub(r'[^a-z0-9]', '', (value or '').strip().lower()) + + @staticmethod + def _normalize_mapping_value(value): + return re.sub(r'\s+', ' ', (value or '').strip()).lower() @staticmethod def _as_float(value): diff --git a/modules/purchase_trade/tests/test_module.py b/modules/purchase_trade/tests/test_module.py index b3e5431..c80c34a 100644 --- a/modules/purchase_trade/tests/test_module.py +++ b/modules/purchase_trade/tests/test_module.py @@ -5190,6 +5190,111 @@ class PurchaseTradeTestCase(ModuleTestCase): self.assertEqual(result[0]['mid_price'], '75.25') self.assertEqual(result[0]['high_price'], '75.25') + @with_transaction() + def test_import_historical_all_sources_uses_mapping_table(self): + 'all sources import maps source columns to standard price values' + ImportPrices = Pool().get('purchase_trade.import_prices') + ns = {'s': 'http://schemas.openxmlformats.org/spreadsheetml/2006/main'} + sheet = ElementTree.fromstring(''' + + + + Copyright + + + combined_x000D_ +description + Original Mid + Original High + Original Low + + + 46191 + 470 + 480 + 460 + + + + ''') + rows = sheet.findall('.//s:sheetData/s:row', ns) + mapping = [ + { + 'excel_tab': 'Argus Download', + 'original_price_index': 'Original Mid', + 'source': 'ARGUS', + 'column_destination': 'mid_price', + 'standard_price_index': 'ARGUS-CFR Brasil', + 'index_description': 'Brazil CFR', + }, + { + 'excel_tab': 'Argus Download', + 'original_price_index': 'Original High', + 'source': 'ARGUS', + 'column_destination': 'high_price', + 'standard_price_index': 'ARGUS-CFR Brasil', + 'index_description': 'Brazil CFR', + }, + { + 'excel_tab': 'Argus Download', + 'original_price_index': 'Original Low', + 'source': 'ARGUS', + 'column_destination': 'low_price', + 'standard_price_index': 'ARGUS-CFR Brasil', + 'index_description': 'Brazil CFR', + }, + { + 'excel_tab': 'Acuity Download', + 'original_price_index': 'Missing Header', + 'source': 'ACUITY', + 'column_destination': 'mid_price', + 'standard_price_index': 'ACUITY-Missing', + 'index_description': 'Missing', + }, + ] + + result = ImportPrices._read_historical_all_sources_rows( + {'Argus Download': rows}, mapping, [], ns) + + self.assertEqual(len(result), 1) + self.assertEqual(result[0]['price_index'], 'ARGUS-CFR Brasil') + self.assertEqual(result[0]['price_date'], '46191') + self.assertEqual(result[0]['mid_price'], '470') + self.assertEqual(result[0]['price_value'], '470') + self.assertEqual(result[0]['high_price'], '480') + self.assertEqual(result[0]['low_price'], '460') + self.assertEqual(result[0]['_source'], 'ARGUS') + self.assertEqual(result[0]['_price_desc'], 'Brazil CFR') + + def test_import_historical_all_sources_requires_exact_tab_match(self): + 'all sources import ignores similarly named source sheets' + ImportPrices = Pool().get('purchase_trade.import_prices') + + result = ImportPrices._matching_sheet_name( + { + 'Acuity Download CODELCO': [], + }, + 'Acuity Download') + + self.assertIsNone(result) + + def test_import_price_index_values_uses_mapping_metadata(self): + 'created price indexes use mapped description and source area' + ImportPrices = Pool().get('purchase_trade.import_prices') + area = Mock(id=8) + + with patch.object( + ImportPrices, '_first_record', + side_effect=lambda model, domain: ( + area if model == 'price.area' else None)): + values = ImportPrices._price_index_values( + 'ARGUS-CFR Brasil', + {'_price_desc': 'Brazil CFR', '_source': 'ARGUS'}) + + self.assertEqual(values['price_index'], 'ARGUS-CFR Brasil') + self.assertEqual(values['price_desc'], 'Brazil CFR') + self.assertEqual(values['price_area'], 8) + def test_pricing_component_matrix_returns_generic_line_price(self): 'matrix pricing can use an unconditional matrix line as a component price' Component = Pool().get('pricing.component') diff --git a/temp_documents/Weekly Forward and historical prices Trade'On.xlsx b/temp_documents/Weekly Forward and historical prices Trade'On.xlsx new file mode 100644 index 0000000..f86cfa0 Binary files /dev/null and b/temp_documents/Weekly Forward and historical prices Trade'On.xlsx differ