feat: Add question-by-question theme analysis

This commit is contained in:
Thomas A. Christensen II 2024-09-04 17:12:54 -05:00
parent c46e55b805
commit 25d9e4aaad
Signed by: millironx
GPG key ID: B7044A3432851F64

55
main.jl
View file

@ -41,7 +41,10 @@ function logged_prompt(prompt)
return response return response
end #function end #function
question_dict = Dict()
for q in questions for q in questions
# Summarize the major themes among all answers
analysis_prompt = """ analysis_prompt = """
The following is a list of answers to a survey with one response per paragraph: The following is a list of answers to a survey with one response per paragraph:
@ -63,9 +66,61 @@ for q in questions
analysis_response = logged_prompt(analysis_prompt) analysis_response = logged_prompt(analysis_prompt)
# Compile all themes that Llama3 identified.
# Llama3 tends to summarize each theme with a bolded statement.
# We will extract the bolded statements to compile themes
themes = String[]
for l in eachline(IOBuffer(analysis_response))
m = match(r"^[1-9]+\. \*\*(.+)\*\*:", l)
isnothing(m) || push!(themes, first(m))
end #for
@info "Found themes $themes"
answer_dict = Dict()
# Now go back through each answer and check if it is addressing the theme noted
for (i, a) in enumerate(skipmissing(survey_data[!, q]))
i == 1 && continue #first "answer" is the question
theme_dict = Dict{String,Union{Bool,Missing}}()
for t in themes
theme_prompt = """
The following was answered as a free-response answer on a survey:
$a
---
Does this answer deal with the theme of $t? Answer yes or no.
"""
theme_response = logged_prompt(theme_prompt)
if startswith(lowercase(theme_response), "yes")
theme_dict[t] = true
elseif startswith(lowercase(theme_response), "no")
theme_dict[t] = false
else
theme_dict[t] = missing
end #if
end #for
answer_dict[a] = theme_dict
end #for
question_dict[q] = answer_dict
end #for end #for
open("results.json", "w") do io
JSON3.write(io, question_dict)
end
# Compile comments from all requested questions # Compile comments from all requested questions
for q in questions for q in questions
open("$q.md", "w") do f open("$q.md", "w") do f