-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(api): Refactor, add validations and generate project score card …
…rating in the data ingestion excel file process
- Loading branch information
Showing
11 changed files
with
365 additions
and
107 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
89 changes: 89 additions & 0 deletions
89
api/src/modules/import/parser/data-ingestion.xlsx-parser.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,89 @@ | ||
| import { RawDataIngestionData } from '@api/modules/import/parser/raw-data-ingestion.type'; | ||
| import { | ||
| ExcelTabNotFoundError, | ||
| IExcelParser, | ||
| } from '@api/modules/import/parser/excel-parser.interface'; | ||
| import { read, utils, WorkBook, WorkSheet } from 'xlsx'; | ||
|
|
||
| export const EXPECTED_SHEETS = [ | ||
| 'Index', | ||
| 'Sheet20', | ||
| 'Pivot Table 1', | ||
| 'Model assumptions and constrain', | ||
| 'Backoffice', | ||
| 'Input', | ||
| 'Model assumptions', | ||
| 'Data pull', | ||
| 'Data ingestion >>>', | ||
| 'master_table', | ||
| 'base_size_table', | ||
| 'base_increase', | ||
| 'Projects', | ||
| 'Base inputs>>>', | ||
| 'Countries', | ||
| 'Ecosystem', | ||
| 'Activity', | ||
| 'Restoration_activity', | ||
| 'Cost inputs>>', | ||
| 'Project size', | ||
| 'Feasibility analysis', | ||
| 'Conservation planning and admin', | ||
| 'Data collection and field costs', | ||
| 'Community representation', | ||
| 'Blue carbon project planning', | ||
| 'Establishing carbon rights', | ||
| 'Financing cost', | ||
| 'Validation', | ||
| 'Implementation labor', | ||
| 'Monitoring', | ||
| 'Maintenance', | ||
| 'Community benefit sharing fund', | ||
| 'Baseline reassessment', | ||
| 'MRV', | ||
| 'Long-term project operating', | ||
| 'Carbon standard fees', | ||
| 'Community cash flow', | ||
| 'Carbon inputs>>', | ||
| 'Ecosystem extent', | ||
| 'Ecosystem loss', | ||
| 'Restorable land', | ||
| 'Sequestration rate', | ||
| 'Emission factors', | ||
| 'Mapping references>>', | ||
| 'Continent', | ||
| 'HDI', | ||
| 'Abbreviation', | ||
| 'Sources>>', | ||
| 'Sequestration rates', | ||
| 'Emissions sources', | ||
| 'MangroveEmissionsValues', | ||
| 'Loss rates & restorable land', | ||
| 'Datasets>>', | ||
| 'Mangrove extent', | ||
| 'Mangrove protected area', | ||
| 'Seagrass extent', | ||
| 'Salt marsh extent', | ||
| 'Mangrove restorable land', | ||
| ] as const; | ||
|
|
||
| export class DataIngestionExcelParser implements IExcelParser { | ||
| public async parseBuffer(buffer: Buffer): Promise<RawDataIngestionData> { | ||
| const workbook: WorkBook = read(buffer); | ||
| const parsedData: any = {}; | ||
|
|
||
| for (const sheetName of EXPECTED_SHEETS) { | ||
| const sheet: WorkSheet = workbook.Sheets[sheetName]; | ||
| if (sheet === undefined) { | ||
| throw new ExcelTabNotFoundError(sheetName); | ||
| } | ||
|
|
||
| const parsedSheet = utils.sheet_to_json(sheet, { | ||
| raw: true, | ||
| }); | ||
| // We can validate the sheet tab headers and column values when we have more information from the science team. | ||
| parsedData[sheetName] = parsedSheet; | ||
| } | ||
|
|
||
| return parsedData; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,21 @@ | ||
| export class ExcelTabNotFoundError extends Error { | ||
| constructor(tabName: string) { | ||
| super(`Tab ${tabName} not found in the excel file`); | ||
| } | ||
| } | ||
|
|
||
| export class ExcelTabHeaderNotFoundError extends Error { | ||
| constructor(header: string, tabName: string) { | ||
| super(`Header (${header}) not found in the tab ${tabName}`); | ||
| } | ||
| } | ||
|
|
||
| export class RowColumnInvalidError extends Error { | ||
| constructor(row: number, column: string) { | ||
| super(`Invalid value in row ${row} and column ${column}`); | ||
| } | ||
| } | ||
|
|
||
| export interface IExcelParser { | ||
| parseBuffer(buffer: Buffer): Promise<unknown>; | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,57 @@ | ||
| import { ExcelBaseIncrease } from '@api/modules/import/dtos/excel-base-increase.dto'; | ||
| import { ExcelBaseSize } from '@api/modules/import/dtos/excel-base-size.dto'; | ||
| import { ExcelBaselineReassessment } from '@api/modules/import/dtos/excel-baseline-reassessment.dto'; | ||
| import { ExcelBlueCarbonProjectPlanning } from '@api/modules/import/dtos/excel-blue-carbon-project-planning.dto'; | ||
| import { ExcelCarbonStandardFees } from '@api/modules/import/dtos/excel-carbon-standard-fees.dto'; | ||
| import { ExcelEcosystemLoss } from '@api/modules/import/dtos/excel-ccosystem-loss.dto'; | ||
| import { ExcelCommunityBenefitSharingFund } from '@api/modules/import/dtos/excel-community-benefit-sharing-fund.dto'; | ||
| import { ExcelCommunityCashFlow } from '@api/modules/import/dtos/excel-community-cash-flow.dto'; | ||
| import { ExcelCommunityRepresentation } from '@api/modules/import/dtos/excel-community-representation.dto'; | ||
| import { ExcelConservationPlanningAndAdmin } from '@api/modules/import/dtos/excel-conservation-planning-and-admin.dto'; | ||
| import { ExcelDataCollectionAndFieldCosts } from '@api/modules/import/dtos/excel-data-collection-field-cost.dto'; | ||
| import { ExcelEcosystemExtent } from '@api/modules/import/dtos/excel-ecosystem-extent.dto'; | ||
| import { ExcelEmissionFactors } from '@api/modules/import/dtos/excel-emission-factors.dto'; | ||
| import { ExcelEstablishingCarbonRights } from '@api/modules/import/dtos/excel-establishing-carbon-rights.dto'; | ||
| import { ExcelFeasibilityAnalysis } from '@api/modules/import/dtos/excel-feasibility-analysis.dto'; | ||
| import { ExcelFinancingCost } from '@api/modules/import/dtos/excel-financing-cost.dto'; | ||
| import { ExcelImplementationLaborCost } from '@api/modules/import/dtos/excel-implementation-labor.dto'; | ||
| import { ExcelLongTermProjectOperating } from '@api/modules/import/dtos/excel-long-term-project-operating.dto'; | ||
| import { ExcelMaintenance } from '@api/modules/import/dtos/excel-maintenance.dto'; | ||
| import { ExcelModelAssumptions } from '@api/modules/import/dtos/excel-model-assumptions.dto'; | ||
| import { ExcelMonitoring } from '@api/modules/import/dtos/excel-monitoring.dto'; | ||
| import { ExcelMRV } from '@api/modules/import/dtos/excel-mrv.dto'; | ||
| import { ExcelProjectSize } from '@api/modules/import/dtos/excel-project-size.dto'; | ||
| import { ExcelProject } from '@api/modules/import/dtos/excel-projects.dto'; | ||
| import { ExcelRestorableLand } from '@api/modules/import/dtos/excel-restorable-land.dto'; | ||
| import { ExcelSequestrationRate } from '@api/modules/import/dtos/excel-sequestration-rate.dto'; | ||
| import { ExcelValidation } from '@api/modules/import/dtos/excel-validation.dto'; | ||
|
|
||
| export type RawDataIngestionData = { | ||
| Projects: ExcelProject[]; | ||
| 'Project size': ExcelProjectSize[]; | ||
| 'Feasibility analysis': ExcelFeasibilityAnalysis[]; | ||
| 'Conservation planning and admin': ExcelConservationPlanningAndAdmin[]; | ||
| 'Data collection and field costs': ExcelDataCollectionAndFieldCosts[]; | ||
| 'Community representation': ExcelCommunityRepresentation[]; | ||
| 'Blue carbon project planning': ExcelBlueCarbonProjectPlanning[]; | ||
| 'Establishing carbon rights': ExcelEstablishingCarbonRights[]; | ||
| 'Financing cost': ExcelFinancingCost[]; | ||
| Validation: ExcelValidation[]; | ||
| Monitoring: ExcelMonitoring[]; | ||
| Maintenance: ExcelMaintenance[]; | ||
| 'Community benefit sharing fund': ExcelCommunityBenefitSharingFund[]; | ||
| 'Baseline reassessment': ExcelBaselineReassessment[]; | ||
| MRV: ExcelMRV[]; | ||
| 'Long-term project operating': ExcelLongTermProjectOperating[]; | ||
| 'Carbon standard fees': ExcelCarbonStandardFees[]; | ||
| 'Community cash flow': ExcelCommunityCashFlow[]; | ||
| 'Ecosystem extent': ExcelEcosystemExtent[]; | ||
| 'Ecosystem loss': ExcelEcosystemLoss[]; | ||
| 'Restorable land': ExcelRestorableLand[]; | ||
| 'Sequestration rate': ExcelSequestrationRate[]; | ||
| 'Emission factors': ExcelEmissionFactors[]; | ||
| 'Implementation labor': ExcelImplementationLaborCost[]; | ||
| base_size_table: ExcelBaseSize[]; | ||
| base_increase: ExcelBaseIncrease[]; | ||
| 'Model assumptions': ExcelModelAssumptions[]; | ||
| }; |
Oops, something went wrong.