Comparing Pandas Vector Operations to Multithreading

# Import necessary libraries and modules
import time
import spacy
import pandas as pd
from concurrent.futures import ThreadPoolExecutor
import utils.data_utils  # Assuming you have a module for loading data

# Load Spacy model
nlp = spacy.load("en_core_web_sm")

# Define NLP preprocessing function
def ner_preprocessing(text):
    doc = nlp(text)
    entities = [ent.text for ent in doc.ents]
    return entities

# Function for parallel processing using ThreadPoolExecutor
def process_transaction_data_parallel(df):
    with ThreadPoolExecutor() as executor:
        processed_data = list(executor.map(ner_preprocessing, df['Description']))
    return processed_data

def main():
    # Load data
    df = utils.data_utils.load_data()

    # Time Pandas vector operation
    start = time.time()
    df['ner_preprocessing_map'] = df['Description'].apply(ner_preprocessing)
    print(f"Time taken for Pandas vector operation: {time.time() - start}")

    # Time multithreading with ThreadPoolExecutor
    start = time.time()
    processed_data = process_transaction_data_parallel(df)
    df['ner_preprocessing_threaded'] = processed_data
    print(f"Time taken for multithreading: {time.time() - start}")

    # Print DataFrame
    print(df.head())

if __name__ == "__main__":
    main()