Percentile matching#
A common method for correcting income under-reporting is matching the income distribution of the survey to an administrative tax dataset.
from loss.loss import Loss, calibration_parameters
from datasets.frs import FRS_2019_20, PercentileMatchedFRS
from datasets.output_dataset import OutputDataset
import torch
import yaml
original_frs = OutputDataset.from_dataset(FRS_2019_20, 2019, 2022)()
percentile_matched_frs = OutputDataset.from_dataset(
PercentileMatchedFRS.from_dataset(
FRS_2019_20,
percentile_matched_variables=["dividend_income"],
force_generate=True,
),
2019,
2022,
)()
loss = Loss(
original_frs,
calibration_parameters(f"2022-01-01"),
static_dataset=False,
)
weights = torch.tensor(original_frs.household.household_weight.values)
frs_loss = loss(weights, original_frs)
percentile_matched_frs_loss = loss(weights, percentile_matched_frs)
print(yaml.dump(loss.computation_tree(weights, percentile_matched_frs)))
Loss.Programs:
1_loss: 1.0213157906954613
2_weight: 1
3_children:
Loss.Programs.DividendIncome:
1_loss: 0.773819055140283
2_weight: 65.2
3_children:
Loss.Programs.DividendIncome.dividend_income_budgetary_impact:
1_loss: 0.5476381102805661
2_weight: 1
3_children:
dividend_income_budgetary_impact_ENGLAND:
1_loss: 2.2063085119521027e-06
2_loss_0: 0.6773687680202487
3_y_pred: 53,320,681,506.21
4_y_0_pred: 9,450,501,623.28
5_y_true: 53,400,000,000.00
dividend_income_budgetary_impact_NORTHERN_IRELAND:
1_loss: 0.9537625500042562
2_loss_0: 0.9537625500042562
3_y_pred: 32,739,489.54
4_y_0_pred: 32,739,489.54
5_y_true: 1,400,000,000.00
dividend_income_budgetary_impact_SCOTLAND:
1_loss: 0.37513747584301366
2_loss_0: 0.6671235232756144
3_y_pred: 6,449,944,820.71
4_y_0_pred: 732,886,634.60
5_y_true: 4,000,000,000.00
dividend_income_budgetary_impact_UNITED_KINGDOM:
1_loss: 0.005990490196922195
2_loss_0: 0.7021411000527296
3_y_pred: 60,153,632,865.29
4_y_0_pred: 10,566,394,796.24
5_y_true: 65,200,000,000.00
dividend_income_budgetary_impact_WALES:
1_loss: 0.680397898556267
2_loss_0: 0.680397898556267
3_y_pred: 350,267,048.82
4_y_0_pred: 350,267,048.82
5_y_true: 2,000,000,000.00
Loss.Programs.IncomeTax:
1_loss: 1.2555149852856258
2_weight: 200
3_children:
Loss.Programs.IncomeTax.IncomeTaxBudgetaryImpact:
1_loss: 1.6544480994919992
2_weight: 1.0
3_children:
income_tax_ENGLAND:
1_loss: 0.003447679807966177
2_loss_0: 0.0006271463139645406
3_y_pred: 192,589,709,802.04
4_y_0_pred: 176,048,741,158.45
5_y_true: 180,994,233,766.23
income_tax_NORTHERN_IRELAND:
1_loss: 1.9256606637561124e-05
2_loss_0: 1.9256606637561124e-05
3_y_pred: 2,955,055,971.07
4_y_0_pred: 2,955,055,971.07
5_y_true: 3,031,870,129.87
income_tax_SCOTLAND:
1_loss: 0.0010639923744253977
2_loss_0: 0.00012708409748636915
3_y_pred: 15,430,047,474.20
4_y_0_pred: 13,283,171,589.62
5_y_true: 13,834,571,428.57
income_tax_UNITED_KINGDOM:
1_loss: 0.0034557659998135045
2_loss_0: 0.0010481049598803094
3_y_pred: 217,051,082,121.98
4_y_0_pred: 198,363,237,593.81
5_y_true: 205,000,000,000.00
income_tax_WALES:
1_loss: 0.0004043363704885881
2_loss_0: 0.0004043363704885881
3_y_pred: 6,076,268,874.68
4_y_0_pred: 6,076,268,874.68
5_y_true: 5,574,935,064.94
income_tax_by_income_0:
1_loss: 6.748906982749366e-05
2_loss_0: 6.748906982749366e-05
3_y_pred: 594,159,126.47
4_y_0_pred: 594,159,126.47
5_y_true: 690,717,092.34
income_tax_by_income_1:
1_loss: 3.0090927382269634e-07
2_loss_0: 3.4312950599093186e-07
3_y_pred: 5,717,438,973.28
4_y_0_pred: 5,718,695,568.22
5_y_true: 5,698,919,449.90
income_tax_by_income_2:
1_loss: 0.00024566560707949735
2_loss_0: 0.0002506845888299809
3_y_pred: 21,544,868,902.65
4_y_0_pred: 21,555,079,012.34
5_y_true: 20,540,275,049.12
income_tax_by_income_3:
1_loss: 0.00015998942257932372
2_loss_0: 0.00018280208234789676
3_y_pred: 40,491,233,395.41
4_y_0_pred: 40,568,586,590.97
5_y_true: 39,368,860,510.81
income_tax_by_income_4:
1_loss: 0.0011024489352243557
2_loss_0: 0.0014657527281342298
3_y_pred: 50,449,270,924.37
4_y_0_pred: 50,943,154,447.18
5_y_true: 47,222,495,088.41
income_tax_by_income_5:
1_loss: 0.002450879594732759
2_loss_0: 0.0022794983690308445
3_y_pred: 19,508,925,168.85
4_y_0_pred: 19,628,073,448.97
5_y_true: 22,856,090,373.28
income_tax_by_income_6:
1_loss: 0.00020941415656882954
2_loss_0: 1.9116027211312415e-05
3_y_pred: 10,785,028,869.52
4_y_0_pred: 11,268,903,161.40
5_y_true: 11,478,388,998.04
income_tax_by_income_7:
1_loss: 0.009937666723533634
2_loss_0: 0.01091179040335241
3_y_pred: 16,459,740,811.64
4_y_0_pred: 16,134,294,355.73
5_y_true: 23,258,840,864.44
income_tax_by_income_8:
1_loss: 0.0670342432366614
2_loss_0: 0.02631339684132812
3_y_pred: 22,791,893,578.77
4_y_0_pred: 3,243,769,566.05
5_y_true: 10,773,575,638.51
income_tax_by_income_9:
1_loss: 0.02539162600358509
2_loss_0: 0.02539162600358509
3_y_pred: 28,657,747,649.16
4_y_0_pred: 28,657,747,649.16
5_y_true: 18,868,860,510.81
Loss.Programs.IncomeTax.IncomeTaxParticipants:
1_loss: 0.8565818710792523
2_weight: 1.0
3_children:
income_tax_payers_ENGLAND_ADDITIONAL:
1_loss: 2.5188396576360072e-05
2_loss_0: 0.0007227589804237184
3_y_pred: 389,032.00
4_y_0_pred: 310,751.00
5_y_true: 407,000.00
income_tax_payers_ENGLAND_BASIC:
1_loss: 0.0011398167436279007
2_loss_0: 0.0010632793647991784
3_y_pred: 21,720,523.00
4_y_0_pred: 21,750,564.00
5_y_true: 22,600,000.00
income_tax_payers_ENGLAND_HIGHER:
1_loss: 2.219227990701562e-06
2_loss_0: 3.8078248690651396e-05
3_y_pred: 3,535,352.00
4_y_0_pred: 3,583,592.00
5_y_true: 3,520,000.00
income_tax_payers_NORTHERN_IRELAND_ADDITIONAL:
1_loss: 1.299108843537418e-07
2_loss_0: 1.299108843537418e-07
3_y_pred: 3,563.00
4_y_0_pred: 3,563.00
5_y_true: 4,000.00
income_tax_payers_NORTHERN_IRELAND_BASIC:
1_loss: 1.5750414639688642e-05
2_loss_0: 1.5750414639688642e-05
3_y_pred: 698,337.00
4_y_0_pred: 698,337.00
5_y_true: 717,000.00
income_tax_payers_NORTHERN_IRELAND_HIGHER:
1_loss: 1.845943877551021e-05
2_loss_0: 1.845943877551021e-05
3_y_pred: 53,275.00
4_y_0_pred: 53,275.00
5_y_true: 60,000.00
income_tax_payers_SCOTLAND_ADDITIONAL:
1_loss: 0.00020802264
2_loss_0: 2.5575834074074068e-05
3_y_pred: 36,758.00
4_y_0_pred: 25,876.00
5_y_true: 20,000.00
income_tax_payers_SCOTLAND_BASIC:
1_loss: 0.00014213282691026316
2_loss_0: 0.00012968087888400265
3_y_pred: 2,073,365.00
4_y_0_pred: 2,077,695.00
5_y_true: 2,170,000.00
income_tax_payers_SCOTLAND_HIGHER:
1_loss: 2.154798316156132e-07
2_loss_0: 5.2835264915082324e-06
3_y_pred: 406,658.00
4_y_0_pred: 413,210.00
5_y_true: 405,000.00
income_tax_payers_WALES_ADDITIONAL:
1_loss: 1.2307000781250008e-05
2_loss_0: 1.2307000781250008e-05
3_y_pred: 9,969.00
4_y_0_pred: 9,969.00
5_y_true: 6,000.00
income_tax_payers_WALES_BASIC:
1_loss: 0.00012816560364064744
2_loss_0: 0.00012816560364064744
3_y_pred: 1,189,844.00
4_y_0_pred: 1,189,844.00
5_y_true: 1,260,000.00
income_tax_payers_WALES_HIGHER:
1_loss: 9.717286352479793e-05
2_loss_0: 9.717286352479793e-05
3_y_pred: 132,756.00
4_y_0_pred: 132,756.00
5_y_true: 113,000.00