PD v1 Pre-Processing block

README.md

@@ -1 +1,11 @@
-**Hello world!!!**
+## Overview
+This block (`block.py`) is responsible for preparing and validating inputs for the model. It performs data cleansing and returns a normalized output dictionary.
+
+## Key Inputs & Outputs
+- **Request**: Refer to `request_schema.json` for detailed input fields and validation rules.  
+- **Response**: Refer to `response_schema.json` for the returned structure and data types.
+
+## Implementation Details
+- All core logic resides in `block.py` within the `__main__` function.
+- Example usage and validation are demonstrated in `test_block.py`.
+

block.py

@@ -1,21 +1,220 @@
-@flowx_block
+import logging
-def example_function(request: dict) -> dict:
+import math
 
-  # Processing logic here...
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(levelname)s] %(name)s - %(message)s",
+)
+logger = logging.getLogger(__name__)
 
-  return {
+def __main__(record_counts_revolving_trade_count:int, record_counts_total_trade_count:int,score_results:float, total_amount_current_balance:float, total_amount_high_credit:float, revolving_amount_credit_limit:float, revolving_amount_percent_available_credit:float, revolving_amount_current_balance:float, revolving_amount_monthly_payment:float, revolving_amount_high_credit:float, closed_with_balance_amount_current_balance:float, closed_with_balance_amount_monthly_payment:float,AGG101:float, AGG102:float, AT09S:int, AT20S:int, AT31S:int, BALMAG01:float, BC21S:int, PAYMNT10:float, REV83:float, US01S:int, monthly_income:float, total_amount_monthly_payment:float, internal_monthly_payment:float)->dict:
-    "meta_info": [
+    
-      {
+    #record_counts_revolving_trade_count treatment
-        "name": "created_date",
+    if record_counts_revolving_trade_count is None or math.isnan(record_counts_revolving_trade_count):
-        "type": "string",
+        record_counts_revolving_trade_count = 0
-        "value": "2024-11-05"
+    elif 0 <= record_counts_revolving_trade_count <= 999:
-      }
+        record_counts_revolving_trade_count = min(30, max(0, record_counts_revolving_trade_count))
-    ],
+    else:
-    "fields": [
+        record_counts_revolving_trade_count = None
-      {
+
-        "name": "",
+    #record_counts_total_trade_count treatment
-        "type": "",
+    if record_counts_total_trade_count is None or math.isnan(record_counts_total_trade_count):
-        "value": ""
+        record_counts_total_trade_count = 0
-      }
+    elif 0 <= record_counts_total_trade_count <= 999:
-    ]
+        record_counts_total_trade_count = min(35, max(0, record_counts_total_trade_count))
+    else:
+        record_counts_total_trade_count = None
+
+    #score_results treatment
+    if score_results is None or math.isnan(score_results):
+        score_results = 595.4689515
+    elif 350.0 <= score_results <= 850.0:
+        score_results = min(700.0, max(541.0, score_results))
+    else:
+        score_results = None
+
+    #revolving_amount_credit_limit treatment
+    if revolving_amount_credit_limit is None or math.isnan(revolving_amount_credit_limit):
+        revolving_amount_credit_limit = 0.0
+    elif 0.0 <= revolving_amount_credit_limit <= 999999999.0:
+        revolving_amount_credit_limit = min(20000.0, max(0.0, revolving_amount_credit_limit))
+    else:
+        revolving_amount_credit_limit = None
+
+    #revolving_amount_percent_available_credit treatment
+    if revolving_amount_percent_available_credit is None or math.isnan(revolving_amount_percent_available_credit):
+        revolving_amount_percent_available_credit = 0.0
+    elif 0.0 <= revolving_amount_percent_available_credit <= 100.0:
+        revolving_amount_percent_available_credit = min(100.0, max(0.0, revolving_amount_percent_available_credit))
+    else:
+        revolving_amount_percent_available_credit = None
+
+    #revolving_amount_current_balance treatment
+    if revolving_amount_current_balance is None or math.isnan(revolving_amount_current_balance):
+        revolving_amount_current_balance = 0.0
+    elif 0.0 <= revolving_amount_current_balance <= 999999999.0:
+        revolving_amount_current_balance = min(20000.0, max(0.0, revolving_amount_current_balance))
+    else:
+        revolving_amount_current_balance = None
+
+    #revolving_amount_monthly_payment treatment
+    if revolving_amount_monthly_payment is None or math.isnan(revolving_amount_monthly_payment):
+        revolving_amount_monthly_payment = 0.0
+    elif 0.0 <= revolving_amount_monthly_payment <= 999999999.0:
+        revolving_amount_monthly_payment = min(500.0, max(0.0, revolving_amount_monthly_payment))
+    else:
+        revolving_amount_monthly_payment = None
+
+    #revolving_amount_high_credit treatment
+    if revolving_amount_high_credit is None or math.isnan(revolving_amount_high_credit):
+        revolving_amount_high_credit = 0.0
+    elif 0.0 <= revolving_amount_high_credit <= 999999999.0:
+        revolving_amount_high_credit = min(20000.0, max(0.0, revolving_amount_high_credit))
+    else:
+        revolving_amount_high_credit = None
+
+    #AGG101 treatment
+    if AGG101 is None or math.isnan(AGG101):
+        AGG101 = 0.0
+    elif 0.0 <= AGG101 <= 999999999.0:
+        AGG101 = min(40000.0, max(0.0, AGG101))
+    else:
+        AGG101 = 0.0
+
+    #AGG102 treatment
+    if AGG102 is None or math.isnan(AGG102):
+        AGG102 = 0.0
+    elif 0.0 <= AGG102 <= 999999999.0:
+        AGG102= min(60000.0, max(0.0, AGG102))
+    else:
+        AGG102 = 0.0
+
+    #AT09S treatment
+    if AT09S is None or math.isnan(AT09S):
+        AT09S = 0
+    elif 0 <= AT09S <= 999:
+        AT09S = min(10, max(0, AT09S))
+    else:
+        AT09S = 0
+
+    #AT20S treatment
+    if AT20S is None or math.isnan(AT20S):
+        AT20S = 0
+    elif 0 <= AT20S <= 999:
+        AT20S = min(500, max(0, AT20S))
+    else:
+        AT20S = 0
+
+    #AT31S treatment
+    if AT31S is None or math.isnan(AT31S):
+        AT31S = 100
+    elif 0 <= AT31S <= 999:
+        AT31S = min(100, max(0, AT31S))
+    else:
+        AT31S = 0
+
+    #BALMAG01 treatment
+    if BALMAG01 is None or math.isnan(BALMAG01):
+        BALMAG01 = 500.0
+    elif 0.0 <= BALMAG01 <= 600.0:
+        BALMAG01 = min(500.0, max(0.0, BALMAG01))
+    else:
+        BALMAG01 = 0.0
+
+    #BC21S treatment
+    if BC21S is None or math.isnan(BC21S):
+        BC21S = 100
+    elif 0 <= BC21S <= 999:
+        BC21S = min(100, max(0, BC21S))
+    else:
+        BC21S = 0
+
+    #PAYMNT10 treatment
+    if PAYMNT10 is None or math.isnan(PAYMNT10):
+        PAYMNT10 = 0.0
+    elif 0.0 <= PAYMNT10 <= 999.0:
+        PAYMNT10 = min(15.0, max(0.0, PAYMNT10))
+    else:
+        PAYMNT10 = 0.0
+
+    #REV83 treatment
+    if REV83 is None or math.isnan(REV83):
+        REV83 = 500.0
+    elif 0.0 <= REV83 <= 999.0:
+        REV83 = min(500.0, max(0.0, REV83))
+    else:
+        REV83 = 0.0
+
+    #US01S treatment
+    if US01S is None or math.isnan(US01S):
+        US01S = 0
+    elif 0 <= US01S <= 999:
+        US01S = min(10, max(0, US01S))
+    else:
+        US01S = 0
+
+    #total_amount_monthly_payment treatment
+    if total_amount_monthly_payment is None or math.isnan(total_amount_monthly_payment):
+        total_amount_monthly_payment = 2382.0
+
+    #internal_monthly_payment treatment
+    if internal_monthly_payment is None or math.isnan(internal_monthly_payment):
+        internal_monthly_payment = 82.27
+
+    #monthly_income treatment
+    if monthly_income is not None and not math.isnan(monthly_income):
+        if 0.0 <= monthly_income <= 999999999.0:
+            monthly_income = min(8500.0, max(0.0, monthly_income))
+    else:
+        monthly_income = None
+
+    #pti calculation
+    if (monthly_income not in [None, 0.0] and not math.isnan(monthly_income) and internal_monthly_payment not in [None, ""] and not math.isnan(internal_monthly_payment) and total_amount_monthly_payment not in [None, ""] and not math.isnan(total_amount_monthly_payment)):
+        pti = (total_amount_monthly_payment + internal_monthly_payment) / monthly_income
+    elif (monthly_income not in [None, 0.0] and not math.isnan(monthly_income) and total_amount_monthly_payment not in [None, ""] and not math.isnan(total_amount_monthly_payment)):
+        pti = (total_amount_monthly_payment + 82.27) / monthly_income
+    else:
+        pti = None
+
+    if total_amount_current_balance is None or math.isnan(total_amount_current_balance):
+        total_amount_current_balance = None
+
+    if total_amount_high_credit is None or math.isnan(total_amount_high_credit):
+        total_amount_high_credit = None
+
+    if closed_with_balance_amount_current_balance is None or math.isnan(closed_with_balance_amount_current_balance):
+        closed_with_balance_amount_current_balance = None
+
+    if closed_with_balance_amount_monthly_payment is None or math.isnan(closed_with_balance_amount_monthly_payment):
+        closed_with_balance_amount_monthly_payment = None
+
+    output_data ={
+        "pti" : pti,
+        "score_results" : score_results,
+        "revolving_amount_monthly_payment" : revolving_amount_monthly_payment,
+        "AT31S" : AT31S,
+        "total_amount_high_credit" : total_amount_high_credit,
+        "AT20S" : AT20S,
+        "BALMAG01" : BALMAG01,
+        "record_counts_revolving_trade_count" : record_counts_revolving_trade_count,
+        "PAYMNT10" : PAYMNT10,
+        "closed_with_balance_amount_current_balance" : closed_with_balance_amount_current_balance,
+        "REV83" : REV83,
+        "AGG102" : AGG102,
+        "BC21S" : BC21S,
+        "record_counts_total_trade_count" : record_counts_total_trade_count,
+        "revolving_amount_current_balance" : revolving_amount_current_balance,
+        "total_amount_current_balance" : total_amount_current_balance,
+        "revolving_amount_high_credit" : revolving_amount_high_credit,
+        "US01S" : US01S,
+        "AGG101" : AGG101,
+        "closed_with_balance_amount_monthly_payment" : closed_with_balance_amount_monthly_payment,
+        "revolving_amount_credit_limit" : revolving_amount_credit_limit,
+        "revolving_amount_percent_available_credit" : revolving_amount_percent_available_credit,
+        "AT09S" : AT09S
     }
+
+    logger.info(f"Pre procesed data of PD V1: {output_data}")
+
+    return output_data

request_schema.json

@@ -1 +1,107 @@
-{}
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": "object",
+  "properties": {
+    "record_counts_revolving_trade_count": {
+      "type": ["integer", "null"],
+      "description": "Records in the database related to revolving trade accounts (a credit card account)"
+    },
+    "record_counts_total_trade_count": {
+      "type": ["integer", "null"],
+      "description": "Total number of trade-related (transaction) records"
+    },
+    "score_results": {
+      "type": ["number", "null"],
+      "description": "TransUnion score"
+    },
+    "total_amount_current_balance": {
+      "type": ["number", "null"],
+      "description": "The total current balance across all credit accounts"
+    },
+    "total_amount_high_credit": {
+      "type": ["number", "null"],
+      "description": "The highest credit amount extended across all credit accounts"
+    },
+    "revolving_amount_credit_limit": {
+      "type": ["number", "null"],
+      "description": "The total credit limit on revolving credit accounts"
+    },
+    "revolving_amount_percent_available_credit": {
+      "type": ["number", "null"],
+      "description": "The percentage of available credit that has been utilized in revolving credit accounts"
+    },
+    "revolving_amount_current_balance": {
+      "type": ["number", "null"],
+      "description": "The current owed balance on revolving credit accounts"
+    },
+    "revolving_amount_monthly_payment": {
+      "type": ["number", "null"],
+      "description": "Minimum amount the borrower is required to pay each month to maintain the account in good standing"
+    },
+    "revolving_amount_high_credit": {
+      "type": ["number", "null"],
+      "description": "The highest credit amount that has been extended to the borrower in revolving credit accounts"
+    },
+    "closed_with_balance_amount_current_balance": {
+      "type": ["number", "null"],
+      "description": "The current balance of closed credit accounts"
+    },
+    "closed_with_balance_amount_monthly_payment": {
+      "type": ["number", "null"],
+      "description": "The monthly payment amount for closed credit accounts (loans)"
+    },
+    "AGG101": {
+      "type": ["number", "null"],
+      "description": "Aggregate non-mortgage balances for month 1"
+    },
+    "AGG102": {
+      "type": ["number", "null"],
+      "description": "Aggregate non-mortgage balances for month 2"
+    },
+    "AT09S": {
+      "type": ["integer", "null"],
+      "description": "Number of trades opened in past 24 months"
+    },
+    "AT20S": {
+      "type": ["integer", "null"],
+      "description": "Months since oldest trade opened"
+    },
+    "AT31S": {
+      "type": ["integer", "null"],
+      "description": "Percentage of open trades > 75% of credit line verified in past 12 months"
+    },
+    "BALMAG01": {
+      "type": ["number", "null"],
+      "description": "Non-mortgage balance magnitude"
+    },
+    "BC21S": {
+      "type": ["integer", "null"],
+      "description": "Months since most recent credit card trade opened"
+    },
+    "PAYMNT10": {
+      "type": ["number", "null"],
+      "description": "Number of payments in the last quarter"
+    },
+    "REV83": {
+      "type": ["number", "null"],
+      "description": "Months since a revolving account last exceeded 75% utilization"
+    },
+    "US01S": {
+      "type": ["integer", "null"],
+      "description": "Number of unsecured installment trades"
+    },
+    "monthly_income": {
+      "type": ["number", "null"],
+      "description": "Monthly income for pti calculation"
+    },
+    "total_amount_monthly_payment": {
+      "type": ["number", "null"],
+      "description": "The total amount of monthly payments across all credit accounts for pti calculation"
+    },
+    "internal_monthly_payment": {
+      "type": ["number", "null"],
+      "description": "If loan provided by Kiwi, expected monthly payment"
+    }
+  },
+  "required": []
+}

requirements.txt

@@ -1 +1 @@
-{}
+jsonschema==4.23.0

response_schema.json

@@ -1 +1,98 @@
-{}
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": "object",
+  "properties": {
+    "pti": {
+      "type": ["number", "null"],
+      "description": "External + internal monthly payment to income ratio"
+    },
+    "score_results": {
+      "type": ["number", "null"],
+      "description": "TransUnion score"
+    },
+    "revolving_amount_monthly_payment": {
+      "type": ["number", "null"],
+      "description": "Minimum amount the borrower is required to pay each month to maintain the account in good standing"
+    },
+    "AT31S": {
+      "type": ["integer", "null"],
+      "description": "Percentage of open trades > 75% of credit line verified in past 12 months"
+    },
+    "total_amount_high_credit": {
+      "type": ["number", "null"],
+      "description": "The highest credit amount extended across all credit accounts"
+    },
+    "AT20S": {
+      "type": ["integer", "null"],
+      "description": "Months since oldest trade opened"
+    },
+    "BALMAG01": {
+      "type": ["number", "null"],
+      "description": "Non-mortgage balance magnitude"
+    },
+    "record_counts_revolving_trade_count": {
+      "type": ["integer", "null"],
+      "description": "Records in the database related to revolving trade accounts (a credit card account)"
+    },
+    "PAYMNT10": {
+      "type": ["number", "null"],
+      "description": "Number of payments in the last quarter"
+    },
+    "closed_with_balance_amount_current_balance": {
+      "type": ["number", "null"],
+      "description": "The current balance of closed credit accounts"
+    },
+    "REV83": {
+      "type": ["number", "null"],
+      "description": "Months since a revolving account last exceeded 75% utilization"
+    },
+    "AGG102": {
+      "type": ["number", "null"],
+      "description": "Aggregate non-mortgage balances for month 2"
+    },
+    "BC21S": {
+      "type": ["integer", "null"],
+      "description": "Months since most recent credit card trade opened"
+    },
+    "record_counts_total_trade_count": {
+      "type": ["integer", "null"],
+      "description": "Total number of trade-related (transaction) records"
+    },
+    "revolving_amount_current_balance": {
+      "type": ["number", "null"],
+      "description": "The current owed balance on revolving credit accounts"
+    },
+    "total_amount_current_balance": {
+      "type": ["number", "null"],
+      "description": "The total current balance across all credit accounts"
+    },
+    "revolving_amount_high_credit": {
+      "type": ["number", "null"],
+      "description": "The highest credit amount that has been extended to the borrower in revolving credit accounts"
+    },
+    "US01S": {
+      "type": ["integer", "null"],
+      "description": "Number of unsecured installment trades"
+    },
+    "AGG101": {
+      "type": ["number", "null"],
+      "description": "Aggregate non-mortgage balances for month 1"
+    },
+    "closed_with_balance_amount_monthly_payment": {
+      "type": ["number", "null"],
+      "description": "The monthly payment amount for closed credit accounts (loans)"
+    },
+    "revolving_amount_credit_limit": {
+      "type": ["number", "null"],
+      "description": "The total credit limit on revolving credit accounts"
+    },
+    "revolving_amount_percent_available_credit": {
+      "type": ["number", "null"],
+      "description": "The percentage of available credit that has been utilized in revolving credit accounts"
+    },
+    "AT09S": {
+      "type": ["integer", "null"],
+      "description": "Number of trades opened in past 24 months"
+    }
+  }
+}

test_block.py

@@ -0,0 +1,36 @@
+import unittest
+from block import __main__
+
+class TestBlock(unittest.TestCase):
+
+    def test_main_success(self):
+        # result = __main__(record_counts_negative_trade_count=7, record_counts_installment_trade_count=8,
+        #                   record_counts_total_trade_count=18, record_counts_total_inquiry_count=3,
+        #                   record_counts_revolving_trade_count=9, score_results=600.0,
+        #                   installment_amount_monthly_payment=572, revolving_amount_percent_available_credit=18,
+        #                   G069S=3, AT24S=6, BR02S=1, BI02S=5, AGG103=27642.0, ALL231=-2.0, AT12S=7, IN20S=166,
+        #                   AT33A=38353, AT35A=5479, AT28A=54087, AT34B=71, S061S=1, RE102S=2000)
+
+        # expected_result = {'score_results': 600.0, 'AT34B': 71, 'AT12S': 7, 'revolving_amount_percent_available_credit': 18, 'AT28A': 54087, 'record_counts_total_trade_count': 18, 'record_counts_negative_trade_count': 7, 'record_counts_revolving_trade_count': 9, 'AT33A': 38353, 'AT35A': 5479, 'record_counts_total_inquiry_count': 3, 'IN20S': 150, 'RE102S': 2000, 'installment_amount_monthly_payment': 572, 'S061S': 1, 'record_counts_installment_trade_count': 8, 'BR02S': 1, 'AGG103': 27642.0, 'ALL231': 0.0, 'G069S': 3, 'AT24S': 6, 'BI02S': 3}
+
+        result = __main__(record_counts_revolving_trade_count =  9.0,record_counts_total_trade_count =  18.0,score_results =  600.0,total_amount_current_balance =  46764.0,total_amount_high_credit =  53807.0,revolving_amount_credit_limit =  2000.0,revolving_amount_percent_available_credit =  18.0,revolving_amount_current_balance =  1635.0,revolving_amount_monthly_payment =  56.0,revolving_amount_high_credit =  1720.0,closed_with_balance_amount_current_balance =  8411.0,closed_with_balance_amount_monthly_payment =  0.0,AGG101 =  11043.0,AGG102 =  24994.0,AT09S =  4.0,AT20S =  166.0,AT31S =  71.0,BALMAG01 =  196.0,BC21S =  4.0,PAYMNT10 =  4.0,REV83 =  0.0,US01S =  0.0,monthly_income = 2200.00,total_amount_monthly_payment =  628.0,internal_monthly_payment =  None)
+
+        expected_result = {'pti': 0.32284999999999997, 'score_results': 600.0, 'revolving_amount_monthly_payment': 56.0, 'AT31S': 71.0, 'total_amount_high_credit': 53807.0, 'AT20S': 166.0, 'BALMAG01': 196.0, 'record_counts_revolving_trade_count': 9.0, 'PAYMNT10': 4.0, 'closed_with_balance_amount_current_balance': 8411.0, 'REV83': 0.0, 'AGG102': 24994.0, 'BC21S': 4.0, 'record_counts_total_trade_count': 18.0, 'revolving_amount_current_balance': 1635.0, 'total_amount_current_balance': 46764.0, 'revolving_amount_high_credit': 1720.0, 'US01S': 0, 'AGG101': 11043.0, 'closed_with_balance_amount_monthly_payment': 0.0, 'revolving_amount_credit_limit': 2000.0, 'revolving_amount_percent_available_credit': 18.0, 'AT09S': 4.0}
+        
+        for key, expected_value in expected_result.items():
+            if isinstance(expected_value, float):
+                self.assertAlmostEqual(result[key], expected_value, places=6, msg=f"Mismatch for {key}")
+            else:
+                self.assertEqual(result[key], expected_value, msg=f"Mismatch for {key}")
+
+    # def test_main_invalid_input(self):
+    #     with self.assertRaises(TypeError):
+    #         __main__(record_counts_negative_trade_count=7, record_counts_installment_trade_count=8,
+    #                  record_counts_total_trade_count=18, record_counts_total_inquiry_count=3,
+    #                  record_counts_revolving_trade_count=9, score_results=600, installment_amount_monthly_payment=572,
+    #                  revolving_amount_percent_available_credit=18, G069S=3, AT24S=6, BR02S=1, BI02S=5,
+    #                  AGG103=27642, ALL231=-2, AT12S=7, IN20S=166, AT33A=38353, AT35A=5479, AT28A=54087,
+    #                  AT34B=71, S061S=1, RE102S=2000)
+
+if __name__ == "__main__":
+    unittest.main()