New file import format (ARGUS +ICIS)

modules/purchase_trade/pricing.py

@@ -83,10 +83,11 @@ class ImportPricesStart(ModelView):
     "Import Prices"
     __name__ = 'purchase_trade.import_prices.start'
 
-    file_structure = fields.Selection([
-        ('historical', "Historical Prices"),
-        ('forward', "Forward Prices"),
-    ], "Excel file structure", required=True)
+    file_structure = fields.Selection([
+        ('historical', "Historical Prices"),
+        ('historical_all_sources', "Historical Prices (All Sources)"),
+        ('forward', "Forward Prices"),
+    ], "Excel file structure", required=True)
     file_ = fields.Binary('Excel file', required=True, filename='filename')
     filename = fields.Char('Filename')
     create_missing_price_index = fields.Boolean(
@@ -123,15 +124,20 @@ class ImportPrices(Wizard):
             Button('OK', 'end', 'tryton-ok', default=True),
         ])
 
-    REQUIRED_COLUMNS = {
-        'priceindex': 'price_index',
-        'pricedate': 'price_date',
+    REQUIRED_COLUMNS = {
+        'priceindex': 'price_index',
+        'pricedate': 'price_date',
         'highprice': 'high_price',
         'lowprice': 'low_price',
         'midprice': 'mid_price',
         'openprice': 'open_price',
         'pricevalue': 'price_value',
-    }
+    }
+    PRICE_INDEX_MAPPING_COLUMNS = (
+        'excel_tab', 'original_price_index', 'source', 'column_destination',
+        'standard_price_index', 'index_description')
+    ALL_SOURCES_VALUE_COLUMNS = {
+        'price_value', 'open_price', 'low_price', 'mid_price', 'high_price'}
 
     def transition_import_(self):
         rows = self._read_xlsx(
@@ -182,16 +188,16 @@ class ImportPrices(Wizard):
                             'missing price_date'))
                     continue
 
-                prices = Price.search(
-                    [('price_index', '=', price_index)], limit=1)
-                if prices:
-                    price = prices[0]
-                elif create_missing_price_index:
-                    price, = Price.create([
-                        cls._price_index_values(price_index)])
-                    stats['created_indexes'].append(
-                        cls._result_line(
-                            row_number, price_index, None,
+                prices = Price.search(
+                    [('price_index', '=', price_index)], limit=1)
+                if prices:
+                    price = prices[0]
+                elif create_missing_price_index:
+                    price, = Price.create([
+                        cls._price_index_values(price_index, row)])
+                    stats['created_indexes'].append(
+                        cls._result_line(
+                            row_number, price_index, None,
                             'price index created'))
                 else:
                     stats['skipped'].append(
@@ -234,21 +240,26 @@ class ImportPrices(Wizard):
         return stats
 
     @classmethod
-    def _price_index_values(cls, price_index):
-        values = {
-            'price_index': price_index,
-            'price_desc': price_index,
-            'price_curve_type': 'future',
-        }
-        references = [
-            ('price_type', 'price.fixtype', [('name', '=', 'Market price')]),
-            ('price_currency', 'currency.currency', [('name', '=', 'USD')]),
-            ('price_calendar', 'price.calendar', [('name', '=', 'Argus EU')]),
-            ('price_unit', 'product.uom', [('name', '=', 'Mt')]),
-        ]
-        for field, model_name, domain in references:
-            record = cls._first_record(model_name, domain)
-            if record:
+    def _price_index_values(cls, price_index, row=None):
+        row = row or {}
+        values = {
+            'price_index': price_index,
+            'price_desc': row.get('_price_desc') or price_index,
+            'price_curve_type': 'future',
+        }
+        references = [
+            ('price_type', 'price.fixtype', [('name', '=', 'Market price')]),
+            ('price_currency', 'currency.currency', [('name', '=', 'USD')]),
+            ('price_calendar', 'price.calendar', [('name', '=', 'Argus EU')]),
+            ('price_unit', 'product.uom', [('name', '=', 'Mt')]),
+        ]
+        source = row.get('_source')
+        if source:
+            references.append(
+                ('price_area', 'price.area', [('name', '=', source)]))
+        for field, model_name, domain in references:
+            record = cls._first_record(model_name, domain)
+            if record:
                 values[field] = record.id
 
         period = cls._period_from_price_index(price_index)
@@ -368,13 +379,16 @@ class ImportPrices(Wizard):
         return ', '.join(parts) or 'price imported'
 
     @classmethod
-    def _read_xlsx(cls, data, file_structure='historical'):
-        try:
-            with zipfile.ZipFile(BytesIO(data)) as workbook:
-                shared_strings = cls._read_shared_strings(workbook)
-                sheet_name = cls._first_sheet_name(workbook)
-                sheet = ElementTree.fromstring(workbook.read(sheet_name))
-        except (KeyError, zipfile.BadZipFile, ElementTree.ParseError):
+    def _read_xlsx(cls, data, file_structure='historical'):
+        try:
+            with zipfile.ZipFile(BytesIO(data)) as workbook:
+                shared_strings = cls._read_shared_strings(workbook)
+                if file_structure == 'historical_all_sources':
+                    return cls._read_historical_all_sources_price_rows(
+                        workbook, shared_strings)
+                sheet_name = cls._first_sheet_name(workbook)
+                sheet = ElementTree.fromstring(workbook.read(sheet_name))
+        except (KeyError, zipfile.BadZipFile, ElementTree.ParseError):
             raise UserError("The selected file is not a valid Excel .xlsx file.")
 
         ns = {'s': 'http://schemas.openxmlformats.org/spreadsheetml/2006/main'}
@@ -382,12 +396,169 @@ class ImportPrices(Wizard):
         if not rows:
             return []
 
-        if file_structure == 'forward':
-            return cls._read_forward_price_rows(rows, shared_strings, ns)
-        return cls._read_historical_price_rows(rows, shared_strings, ns)
-
-    @classmethod
-    def _read_historical_price_rows(cls, rows, shared_strings, ns):
+        if file_structure == 'forward':
+            return cls._read_forward_price_rows(rows, shared_strings, ns)
+        return cls._read_historical_price_rows(rows, shared_strings, ns)
+
+    @classmethod
+    def _read_historical_all_sources_price_rows(cls, workbook, shared_strings):
+        ns = {'s': 'http://schemas.openxmlformats.org/spreadsheetml/2006/main'}
+        sheet_rows = cls._workbook_sheet_rows(workbook, ns)
+        mapping = cls._price_index_mapping()
+        return cls._read_historical_all_sources_rows(
+            sheet_rows, mapping, shared_strings, ns)
+
+    @classmethod
+    def _read_historical_all_sources_rows(
+            cls, sheet_rows, mapping, shared_strings, ns):
+        result_by_key = {}
+        mapping_by_tab = {}
+        for mapping_row in mapping:
+            tab = mapping_row.get('excel_tab')
+            destination = mapping_row.get('column_destination')
+            if not tab or destination not in cls.ALL_SOURCES_VALUE_COLUMNS:
+                continue
+            mapping_by_tab.setdefault(tab, []).append(mapping_row)
+
+        for tab, tab_mapping in mapping_by_tab.items():
+            sheet_name = cls._matching_sheet_name(sheet_rows, tab)
+            if not sheet_name:
+                continue
+            rows = sheet_rows[sheet_name]
+            header_row_number, mapped_columns = cls._mapped_columns(
+                rows, tab_mapping, shared_strings, ns)
+            if not mapped_columns:
+                continue
+            date_column = cls._date_column(
+                rows[header_row_number - 1], mapped_columns, shared_strings,
+                ns)
+
+            for excel_row_number, sheet_row in enumerate(
+                    rows[header_row_number:], start=header_row_number + 1):
+                row_values = cls._row_values(sheet_row, shared_strings, ns)
+                price_date = row_values.get(date_column)
+                if price_date in (None, ''):
+                    continue
+                for column, mapping_row in mapped_columns.items():
+                    price_value = row_values.get(column)
+                    if price_value in (None, ''):
+                        continue
+                    price_index = (
+                        mapping_row.get('standard_price_index') or '').strip()
+                    if not price_index:
+                        continue
+                    key = (price_index, price_date, excel_row_number)
+                    result = result_by_key.setdefault(key, {
+                        '_row_number': excel_row_number,
+                        '_source': mapping_row.get('source'),
+                        '_price_desc': mapping_row.get('index_description'),
+                        'price_index': price_index,
+                        'price_date': price_date,
+                    })
+                    result[mapping_row['column_destination']] = price_value
+                    if (mapping_row['column_destination'] == 'mid_price'
+                            and not result.get('price_value')):
+                        result['price_value'] = price_value
+        return list(result_by_key.values())
+
+    @classmethod
+    def _price_index_mapping(cls):
+        cursor = Transaction().connection.cursor()
+        columns = ', '.join(cls.PRICE_INDEX_MAPPING_COLUMNS)
+        cursor.execute(
+            'SELECT %s FROM public."ext_H2SO4_PriceIndexMapping"'
+            % columns)
+        return [
+            dict(zip(cls.PRICE_INDEX_MAPPING_COLUMNS, row))
+            for row in cursor.fetchall()]
+
+    @classmethod
+    def _mapped_columns(cls, rows, mapping, shared_strings, ns):
+        mapping_by_header = {}
+        for mapping_row in mapping:
+            header = cls._normalize_mapping_value(
+                mapping_row.get('original_price_index'))
+            mapping_by_header[header] = mapping_row
+
+        hits_by_row = {}
+        columns_by_row = {}
+        for row_number, sheet_row in enumerate(rows, start=1):
+            for cell in sheet_row.findall('s:c', ns):
+                header = cls._normalize_mapping_value(
+                    cls._cell_value(cell, shared_strings))
+                mapping_row = mapping_by_header.get(header)
+                if not mapping_row:
+                    continue
+                column = cls._cell_column_index(cell.get('r'))
+                hits_by_row[row_number] = hits_by_row.get(row_number, 0) + 1
+                columns_by_row.setdefault(row_number, {})[column] = mapping_row
+        if not hits_by_row:
+            return None, {}
+        header_row_number = max(hits_by_row, key=hits_by_row.get)
+        return header_row_number, columns_by_row[header_row_number]
+
+    @classmethod
+    def _date_column(cls, header_row, mapped_columns, shared_strings, ns):
+        for cell in header_row.findall('s:c', ns):
+            if cls._normalize_header(
+                    cls._cell_value(cell, shared_strings)) == 'date':
+                return cls._cell_column_index(cell.get('r'))
+        return min(mapped_columns) - 1
+
+    @classmethod
+    def _row_values(cls, sheet_row, shared_strings, ns):
+        values = {}
+        for cell in sheet_row.findall('s:c', ns):
+            values[cls._cell_column_index(cell.get('r'))] = cls._cell_value(
+                cell, shared_strings)
+        return values
+
+    @classmethod
+    def _workbook_sheet_rows(cls, workbook, ns):
+        sheet_rows = {}
+        for sheet_name, sheet_path in cls._worksheet_paths(workbook):
+            sheet = ElementTree.fromstring(workbook.read(sheet_path))
+            sheet_rows[sheet_name] = sheet.findall('.//s:sheetData/s:row', ns)
+        return sheet_rows
+
+    @classmethod
+    def _worksheet_paths(cls, workbook):
+        workbook_xml = ElementTree.fromstring(workbook.read('xl/workbook.xml'))
+        rels_xml = ElementTree.fromstring(
+            workbook.read('xl/_rels/workbook.xml.rels'))
+        wb_ns = {
+            's': 'http://schemas.openxmlformats.org/spreadsheetml/2006/main',
+            'r': (
+                'http://schemas.openxmlformats.org/officeDocument/2006/'
+                'relationships'),
+        }
+        rel_ns = {
+            'r': (
+                'http://schemas.openxmlformats.org/package/2006/'
+                'relationships'),
+        }
+        rels = {
+            rel.get('Id'): rel.get('Target')
+            for rel in rels_xml.findall('r:Relationship', rel_ns)}
+        for sheet in workbook_xml.findall('.//s:sheet', wb_ns):
+            rel_id = sheet.get(
+                '{http://schemas.openxmlformats.org/officeDocument/2006/'
+                'relationships}id')
+            target = rels[rel_id]
+            if not target.startswith('/'):
+                target = 'xl/' + target
+            yield sheet.get('name'), target.lstrip('/')
+
+    @classmethod
+    def _matching_sheet_name(cls, sheet_rows, mapping_tab):
+        normalized_tab = cls._normalize_mapping_value(mapping_tab)
+        for sheet_name in sheet_rows:
+            if cls._normalize_mapping_value(sheet_name) == normalized_tab:
+                return sheet_name
+        return None
+
+    @classmethod
+    def _read_historical_price_rows(cls, rows, shared_strings, ns):
         headers = {}
         for cell in rows[0].findall('s:c', ns):
             index = cls._cell_column_index(cell.get('r'))
@@ -537,8 +708,12 @@ class ImportPrices(Wizard):
         return value.text
 
     @staticmethod
-    def _normalize_header(value):
-        return re.sub(r'[^a-z0-9]', '', (value or '').strip().lower())
+    def _normalize_header(value):
+        return re.sub(r'[^a-z0-9]', '', (value or '').strip().lower())
+
+    @staticmethod
+    def _normalize_mapping_value(value):
+        return re.sub(r'\s+', ' ', (value or '').strip()).lower()
 
     @staticmethod
     def _as_float(value):

modules/purchase_trade/tests/test_module.py

@@ -5190,6 +5190,111 @@ class PurchaseTradeTestCase(ModuleTestCase):
         self.assertEqual(result[0]['mid_price'], '75.25')
         self.assertEqual(result[0]['high_price'], '75.25')
 
+    @with_transaction()
+    def test_import_historical_all_sources_uses_mapping_table(self):
+        'all sources import maps source columns to standard price values'
+        ImportPrices = Pool().get('purchase_trade.import_prices')
+        ns = {'s': 'http://schemas.openxmlformats.org/spreadsheetml/2006/main'}
+        sheet = ElementTree.fromstring('''
+            <worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">
+              <sheetData>
+                <row r="1">
+                  <c r="A1" t="inlineStr"><is><t>Copyright</t></is></c>
+                </row>
+                <row r="2">
+                  <c r="A2" t="inlineStr"><is><t>combined_x000D_
+description</t></is></c>
+                  <c r="B2" t="inlineStr"><is><t>Original Mid</t></is></c>
+                  <c r="C2" t="inlineStr"><is><t>Original High</t></is></c>
+                  <c r="D2" t="inlineStr"><is><t>Original Low</t></is></c>
+                </row>
+                <row r="3">
+                  <c r="A3"><v>46191</v></c>
+                  <c r="B3"><v>470</v></c>
+                  <c r="C3"><v>480</v></c>
+                  <c r="D3"><v>460</v></c>
+                </row>
+              </sheetData>
+            </worksheet>
+        ''')
+        rows = sheet.findall('.//s:sheetData/s:row', ns)
+        mapping = [
+            {
+                'excel_tab': 'Argus Download',
+                'original_price_index': 'Original Mid',
+                'source': 'ARGUS',
+                'column_destination': 'mid_price',
+                'standard_price_index': 'ARGUS-CFR Brasil',
+                'index_description': 'Brazil CFR',
+            },
+            {
+                'excel_tab': 'Argus Download',
+                'original_price_index': 'Original High',
+                'source': 'ARGUS',
+                'column_destination': 'high_price',
+                'standard_price_index': 'ARGUS-CFR Brasil',
+                'index_description': 'Brazil CFR',
+            },
+            {
+                'excel_tab': 'Argus Download',
+                'original_price_index': 'Original Low',
+                'source': 'ARGUS',
+                'column_destination': 'low_price',
+                'standard_price_index': 'ARGUS-CFR Brasil',
+                'index_description': 'Brazil CFR',
+            },
+            {
+                'excel_tab': 'Acuity Download',
+                'original_price_index': 'Missing Header',
+                'source': 'ACUITY',
+                'column_destination': 'mid_price',
+                'standard_price_index': 'ACUITY-Missing',
+                'index_description': 'Missing',
+            },
+        ]
+
+        result = ImportPrices._read_historical_all_sources_rows(
+            {'Argus Download': rows}, mapping, [], ns)
+
+        self.assertEqual(len(result), 1)
+        self.assertEqual(result[0]['price_index'], 'ARGUS-CFR Brasil')
+        self.assertEqual(result[0]['price_date'], '46191')
+        self.assertEqual(result[0]['mid_price'], '470')
+        self.assertEqual(result[0]['price_value'], '470')
+        self.assertEqual(result[0]['high_price'], '480')
+        self.assertEqual(result[0]['low_price'], '460')
+        self.assertEqual(result[0]['_source'], 'ARGUS')
+        self.assertEqual(result[0]['_price_desc'], 'Brazil CFR')
+
+    def test_import_historical_all_sources_requires_exact_tab_match(self):
+        'all sources import ignores similarly named source sheets'
+        ImportPrices = Pool().get('purchase_trade.import_prices')
+
+        result = ImportPrices._matching_sheet_name(
+            {
+                'Acuity Download CODELCO': [],
+            },
+            'Acuity Download')
+
+        self.assertIsNone(result)
+
+    def test_import_price_index_values_uses_mapping_metadata(self):
+        'created price indexes use mapped description and source area'
+        ImportPrices = Pool().get('purchase_trade.import_prices')
+        area = Mock(id=8)
+
+        with patch.object(
+                ImportPrices, '_first_record',
+                side_effect=lambda model, domain: (
+                    area if model == 'price.area' else None)):
+            values = ImportPrices._price_index_values(
+                'ARGUS-CFR Brasil',
+                {'_price_desc': 'Brazil CFR', '_source': 'ARGUS'})
+
+        self.assertEqual(values['price_index'], 'ARGUS-CFR Brasil')
+        self.assertEqual(values['price_desc'], 'Brazil CFR')
+        self.assertEqual(values['price_area'], 8)
+
     def test_pricing_component_matrix_returns_generic_line_price(self):
         'matrix pricing can use an unconditional matrix line as a component price'
         Component = Pool().get('pricing.component')