alexander/whu-textual-analysis
1
0
Fork 0
Code Activity
whu-textual-analysis/exam/part2_problem1/problem1_code.py

500 lines
19 KiB
Python
Raw Permalink Blame History

# -*- coding: utf-8 -*-
"""
Created on Sat Jul 30 15:20:32 2022
@author: Alexander Hillert, Goethe University
"""
# import packages
import re
import textwrap
# define working directory
# adjust it to your computer
directory = "/home/alexander/repos/whu-textual-analysis/exam/part2_problem1/"
# =============================================================================
# Part A: Creating an Overview File on the Call Participants
# =============================================================================
# Create output file
output_csv_file = open(
directory + "Problem_1_Overview_Calls.csv", "w", encoding="utf-8"
)
# Write variable names to the first line of the output file
# 1) Call-ID
# 2) Filename
# 3) Fiscal Quarter
# 4) Fiscal Year
# 5) Date of the call in the format YYYYMMDD
# 6) Time of the call, e.g., 05:00 PM GMT
# 7) number of non-corporate call participants
# 8) the names of all corporate participants and their positions -> each item
# should be written in a seperate column
output_csv_file.write("ID;Filename;Fiscal_Quarter;Fiscal_Year;Date;Time;Analysts")
# There can be up to 4 corporate particiapnts
for i in range(1, 5):
output_csv_file.write(";Name_" + str(i) + ";Position_" + str(i))
output_csv_file.write("\n")
# Open the overfiew file "Overview_File_Problem_1.csv" to call the earnings calls
overview_file = open(directory + "Overview_File_Problem_1.csv", "r", encoding="utf-8")
overview_text = overview_file.read()
list_earnings_calls = overview_text.split("\n")
# The last line is empty -> drop it
while list_earnings_calls.count("") > 0:
list_earnings_calls.remove("")
# iterate all earnings conference calls
for i in range(1, len(list_earnings_calls)):
# reset the variables
fiscal_quarter = ""
fiscal_year = ""
date = ""
time = ""
# we split the entire transcripts into three parts
# its header
header_text = ""
# the list of non-corporate participants
participants_text = ""
# the list of corporate participants
corporates_text = ""
# the number of analysts joining the call
number_analysts = 0
# variables for manager name and position
manager_name = ""
manager_position = ""
manager_position_edited = ""
# a list of manager names for part b)
manager_name_list = []
# get the filename of each earnings call
call_information = list_earnings_calls[i].split(";")
call_id = call_information[0]
filename = call_information[1]
# open the call transcript
call_file = open(directory + "Problem_1_Sample/" + filename, "r", encoding="utf-8")
call_text = call_file.read()
# Get information on the call
# FOr example:
# Q1 2013 Bank of America Corporation Earnings Conference Call
# 04/17/2013 08:30 AM GMT
# the header ends where the list of corporate particpants starts
splitter = textwrap.dedent(
"""
================================================================================
Corporate Participants
================================================================================
"""
).strip()
match_corporates = re.search(splitter, call_text)
if match_corporates:
header_text = call_text[: match_corporates.start()].strip()
else: # sanity check to verify no record ends up here
raise RuntimeError("could not split")
# get the fiscal quarter and year from the header text
match_fiscal_quarter = re.search("Q([1-4]{1}) ([0-9]{4})", header_text)
if match_fiscal_quarter:
fiscal_quarter = match_fiscal_quarter.group(1)
fiscal_year = match_fiscal_quarter.group(2)
else: # sanity check to verify no record ends up here
raise RuntimeError("could not split")
# get date and time of the call
match_date = re.search(
"([0-9]{2}/[0-9]{2}/[0-9]{4}) ([0-9]{2}:[0-9]{2} [A-Z]{2}) [A-Z]", header_text
)
if match_date:
# date
date = match_date.group(1)
# the date in the output file should be formatted as YYYYMMDD
# so, you need to rearrange the date text
year = date[-4:]
month = date[3:-5]
day = date[:2]
date_formatted = year + month + day
# time
time = match_date.group(2)
else: # sanity check to verify no record ends up here
raise RuntimeError("could not split")
# count the number of analysts
# the relevant text part starts with, for example,
# ================================================================================
# Conference Call Participiants
# ================================================================================
#
# * Chris Mutascio
# Keefe, Bruyette & Woods - Analyst
# * Thomas Laturneau
# FBR - Analyst
# and ends with the beginning of the presentation
# ================================================================================
# Presentation
# --------------------------------------------------------------------------------
splitter1 = textwrap.dedent(
"""
================================================================================
Conference Call Participiants
================================================================================
"""
).strip()
splitter2 = textwrap.dedent(
"""
================================================================================
Presentation
"""
).strip()
match_participants = re.search(splitter1, call_text)
match_presentation = re.search(splitter2, call_text)
# if you find both boundaries
if match_participants and match_presentation:
# get the text in between
participants_text = call_text[
match_participants.end() : match_presentation.start()
]
else: # sanity check to verify no record ends up here
raise RuntimeError("could not split")
# split the text of the participants that you have just identified
# in a way that each element refers to one analyst.
analyst_list = [
x.replace("\n ", ", ").strip() for x in participants_text.split(" * ")
]
# depending on how you split, you might need re.split()
# check whether you get empty elements and/or elements that do not refer
# to analysts -> remove them
while analyst_list.count("") > 0:
analyst_list.remove("")
# after these steps and checks, the number of analysts is the length of your analyst list
number_analysts = len(analyst_list)
# get the names of the corporate participants and their position
# remember that you already have the beginning of corporate participants
# see above at around line 90
# the corporate participants come before the list of non-corporate participants
corporates_text = call_text[match_corporates.end() : match_participants.start()]
# like before, split this text such that one element refers to one corporate participant
corporates_list = [
x.replace("\n ", ", ").strip() for x in corporates_text.split(" * ")
]
# check whether you get empty elements and/or elements that do not refer
# to corporate participants -> remove them
while corporates_list.count("") > 0:
corporates_list.remove("")
# write the call information to the output file
output_csv_file.write(
str(call_id)
+ ";"
+ filename
+ ";"
+ fiscal_quarter
+ ";"
+ fiscal_year
+ ";"
+ date_formatted
+ ";"
+ time
+ ";"
+ str(number_analysts)
)
# now, we need to add the information on the corporate participants
# go over all corporate participants
for j in range(len(corporates_list)):
# depending on how you split the text of corporate participants,
# one element of your list could contain the name of the mangager in the first
# line and their position in the second line.
# ADJUST THE FOLLOWING COMMANDS IF YOU USED A DIFFERENT SPLIT.
# split each element of the list of corporate participants further
# into name and position
manager_entry = corporates_list[j]
manager_entry_parts = manager_entry.split(", ")
manager_name = manager_entry_parts[0]
# for part b) of the problem it is helpful to have a list of all
# manager names. With this list, we can identify whether a statement
# comes from a managers (-> answer) or from an analyst (-> question)
manager_name_list.append(manager_name)
manager_entry_parts = manager_entry.split(" - ")
manager_position = manager_entry_parts[-1]
output_csv_file.write(";" + manager_name + ";" + manager_position)
output_csv_file.write("\n")
print("For earnings call " + str(i) + " part a) has been completed.")
# =========================================================================
# Part B: Extracting the Call Segments
# =========================================================================
# set variables
presentation_text = ""
qanda_text = ""
qanda_list = []
question_text = ""
answer_text = ""
# identify the presentation
# the begin of the presentation has already been identified above
# see at around line 140
#
# the presentation ends where the Q and A part begins
# ================================================================================
# Questions and Answers
# --------------------------------------------------------------------------------
splitter = textwrap.dedent(
"""
================================================================================
Questions and Answers
"""
).strip()
match_qanda = re.search(splitter, call_text)
presentation_text = call_text[match_presentation.end() : match_qanda.start()]
# drop operator statements
# search for the beginning of an operator statement
splitter1 = textwrap.dedent(
"""
--------------------------------------------------------------------------------
Operator \[.*\]
--------------------------------------------------------------------------------
"""
).strip()
splitter2 = textwrap.dedent(
"""
--------------------------------------------------------------------------------
"""
).strip()
match_operator = re.search(splitter1, presentation_text)
while match_operator:
match_operator_start = match_operator.start()
# search for the end of the operator statement
# Hint: search only after the beginning of the operator statement
# Hint 2: remember to keep track of your coordinates (.start() and .end())
match_operator_end = re.search(
splitter2, presentation_text[match_operator.end() :]
)
match_operator_end = match_operator_end.start() + match_operator.end()
# keep the text before the operator statement and the text after
# the approach is similar to removing tables (see Problem 4 and 5 from class)
presentation_text = (
presentation_text[:match_operator_start]
+ presentation_text[match_operator_end:]
)
# check whether there is another match
match_operator = re.search(splitter1, presentation_text)
# sometimes there are technical remarks like "(inaudible)", "(corrected by company after the call)",
# or "(technical difficulty)" -> drop those
# there are several ways to approach this editing step (e.g., re.sub())
presentation_text = re.sub("\(.*\)", "", presentation_text)
# drop information on the speakers, e.g.,
# -------------------------------------------------------------------------
# Deborah Crawford, Facebook, Inc. - Director of IR [2]
# -------------------------------------------------------------------------
splitter = textwrap.dedent(
"""
--------------------------------------------------------------------------------
.*, .* - .* \[.*\]
--------------------------------------------------------------------------------
"""
).strip()
match_speaker = re.search(splitter, presentation_text)
while match_speaker:
# the task is similar to the Operator statement but be careful
# to only remove the speaker name but NOT the text of the speaker.
presentation_text = (
presentation_text[: match_speaker.start()]
+ presentation_text[match_speaker.end() :].strip()
)
# check whether there is another speaker name
match_speaker = re.search(splitter, presentation_text)
# write the text of the presentation to an output file
# make sure that the folder "Problem_1_Conference_Call_Segments" exists.
output_file_presentation = open(
directory
+ "Problem_1_Conference_Call_Segments/call_"
+ str(call_id)
+ "_presentation.txt",
"w",
encoding="utf-8",
)
output_file_presentation.write(presentation_text)
# Close file
output_file_presentation.close()
# -------------------------------------------------------------------------
# identify questions and answers
# -------------------------------------------------------------------------
# you already have the start of the Q&A section (see at around lines 235)
qanda_text = call_text[match_qanda.end() :]
# the earnings call transcript ends with definitions
# remove these/keep the text before the definitions
splitter = textwrap.dedent(
"""
--------------------------------------------------------------------------------
Definitions
--------------------------------------------------------------------------------
"""
).strip()
match_definitions = re.search(splitter, qanda_text)
if match_definitions:
# keep the text before
qanda_text = qanda_text[: match_definitions.start()]
else: # sanity check to verify no record ends up here
raise RuntimeError("could not split")
# split the Q and A part by speaker
# -10 trick
qanda_list = re.split("\n\n----------", qanda_text)
# variables to count the number of answers
answer_counter = 1
# and questions
question_counter = 1
# go over all speakers/statements that you obtained from the previous split
# you now have to decide whether the speaker is an analyst (-> question)
# or a corporate participant (-> answer)
for k in range(len(qanda_list)):
# identify the speaker name to check whether it is a corporate participant.
# For example
# --------------------------------------------------------------------------------
# Bruce Thompson, Bank of America Corporation - CFO [3]
# --------------------------------------------------------------------------------
#
speaker_text_part = qanda_list[k]
# split the text part of the kth speaker
# into his*her name and the rest
# NOTE: re.search() and re.sub() are also nice ways to accomplish the goal
splitter = textwrap.dedent(
"""
----------------------------------------------------------------------
(.*),? .* \[.*\]
--------------------------------------------------------------------------------
(.*)
"""
).strip()
match_speaker = re.search(splitter, speaker_text_part)
if match_speaker:
# get the name of the speaker from the previous split
# in the example above, we need to get "Bruce Thompson"
speaker_name = match_speaker.group(1).strip()
# edge case
if "," in speaker_name:
speaker_name = speaker_name.split(",")[0]
# depending on your split, you might need some further editing to
# get onyl the name ("Bruce Thompson") without any additional information.
# Drop Operator Statements
if "Operator" in speaker_name:
continue
else: # sanity check to verify no record ends up here
raise RuntimeError("could not split")
# the second part of speaker_text_sub_parts is (probably) the statement
# of the speaker (again, it depends on your split)
text = speaker_text_part.split(
"--------------------------------------------------------------------------------"
)[-1].strip()
# sometimes there are technical remarks like "(inaudible)", "(corrected by company after the call)",
# or "(technical difficulty)" -> drop those
text = re.sub("\(.*\)", "", text)
# there are several ways to approach this editing step (e.g., re.sub())
# check whether the speaker name is in the manager list from part a) (see at around line 195)
if speaker_name in manager_name_list:
# the name of the speaker is in the list of corporate participants
# -> it is a management answer
answer_text = (
answer_text
+ "Answer_"
+ str(answer_counter)
+ ":\n\n "
+ text
+ "\n\n"
) # adjusted to make it look like problem 2's structure
answer_counter = answer_counter + 1
else:
# it is either an analyst question or an operator statement
# be careful to check the condition below. depending on how your
# speaker names look like, you may need .count() and/or re.search() instead of .startswith()
if speaker_name.startswith(
"Operator"
): # or TO BE COMPLETED: ALREADY HANDLED ABOVE
pass
else:
# it is an analyst question
question_text = (
question_text
+ "Question_"
+ str(question_counter)
+ ":\n\n "
+ text
+ "\n\n"
) # adjusted to make it look like problem 2's structure
question_counter = question_counter + 1
# write the texts to output files
# make sure that the subfolder exists.
output_file_answers = open(
directory
+ "Problem_1_Conference_Call_Segments/call_"
+ str(call_id)
+ "_answers.txt",
"w",
encoding="utf-8",
)
output_file_questions = open(
directory
+ "Problem_1_Conference_Call_Segments/call_"
+ str(call_id)
+ "_questions.txt",
"w",
encoding="utf-8",
)
output_file_answers.write(answer_text)
output_file_questions.write(question_text)
# Close files
output_file_answers.close()
output_file_questions.close()
call_file.close()
# Close files
overview_file.close()
output_csv_file.close()
print("Problem 1 completed.")
View git blame Copy permalink