daily gospel fetcher
Hardware_Log / MDX
Hardware Used
Microcontroller : ESP32
- Brand : Inland ESP32-WROOM-32D Module
Initial Setup
// INITIAL_PROVISIONING_UI
If I am going to make a few of these and hand them out to friends, there needs to be a way for the board to connect to their WiFi and not just my own that I hardcoded to connect to. Hence the WiFi setup Process!
Since this project has minimal hardware input (only a rotary encoder), it unfortunately can be a bit clunky to setup and manually input the Username/Password. Luckily, the ESP32 saves the last successfully connected WiFi network to flash storage, so by default we will always try to connect to this saved network first before guiding the user through the setup process.
Fetching Process
Now that we are connected to the internet, we can start fetching the readings from USCCB.org!
I cound not find any pre-made community APIs for doing a simple call to get the readings, but I knew that the readings were readily available on the USCCB website for any date. This provides a perfect place that the readings can be scraped from given its consistent format and structure.
// [title: scraping_engine.ino]
void fetchAllReadings() {
if (WiFi.status() == WL_CONNECTED) {
WiFiClientSecure client;
client.setInsecure(); // Ignore SSL certificate errors
HTTPClient http;
// Set a timeout for the connection
http.setTimeout(10000);
http.setUserAgent("Mozilla/5.0 (ESP32)");
Serial.println("Starting HTTP GET...");
// formattedTimeNum = "012626"; // e.g., "012526" - Test Date
if (http.begin(client, serverUrl+formattedTimeNum+".cfm")) { // Full URL with date
int httpCode = http.GET();
Serial.print("HTTP Code: ");
Serial.println(httpCode);
if (httpCode == HTTP_CODE_OK) {
WiFiClient *stream = http.getStreamPtr();
String currentTagName = "";
bool lookingForAddress = false;
// Timeout variables for the stream loop
unsigned long lastByteTime = millis();
const unsigned long TIMEOUT_MS = 5000;
Serial.println("Parsing Stream...");
while (http.connected() && (stream->available() > 0 || stream->connected())) {
// SAFETY: Break if stream hangs for 5 seconds
if (millis() - lastByteTime > TIMEOUT_MS) {
Serial.println("Error: Stream timed out.");
break;
}
if (stream->available()) {
lastByteTime = millis(); // Reset timeout timer
String line = stream->readStringUntil('\n');
line.trim();
// --- STATE 1: FIND NAME (e.g., "Gospel") ---
if (!lookingForAddress) {
if (line.indexOf("<h3 class=\"name\">") >= 0) {
// Extract name between tags
int start = line.indexOf(">") + 1;
int end = line.indexOf("</h3>");
currentTagName = line.substring(start, end);
Serial.println("--------------------------------");
Serial.print("FOUND SECTION: ");
Serial.println(currentTagName);
lookingForAddress = true;
}
}
// --- STATE 2: FIND ADDRESS (e.g., "John 3:16") ---
else {
if (line.indexOf("div class=\"address\"") >= 0) {
Serial.println(" -> Found Address Div! Scanning multi-line...");
// 1. Capture FULL HTML BLOCK (Fixes blank string issue)
String rawHtml = line;
int safetyCount = 0;
// Keep reading lines until we find the closing div
while (rawHtml.indexOf("</div>") == -1 && safetyCount < 20) {
if (stream->available()) {
String nextLine = stream->readStringUntil('\n');
rawHtml += " " + nextLine;
safetyCount++;
} else {
delay(10);
}
}
// 2. Strip HTML Tags manually
String cleanAddress = "";
bool insideTag = false;
for (unsigned int i=0; i < rawHtml.length(); i++) {
char c = rawHtml[i];
if (c == '<') {
insideTag = true;
} else if (c == '>') {
insideTag = false;
} else if (!insideTag) {
// Filter: Only allow visible characters (ASCII 32-126)
if (c >= 32 && c <= 126) {
cleanAddress += c;
}
}
}
cleanAddress.trim(); // Remove leading/trailing spaces
Serial.print(" -> Cleaned Text: '");
Serial.print(cleanAddress);
Serial.println("'");
// 3. Save to memory
if (cleanAddress.length() > 0 && readingCount < 5 && currentTagName != "Alleluia") {
dailyReadings[readingCount].name = currentTagName;
dailyReadings[readingCount].address = cleanAddress;
readingCount++;
}
lookingForAddress = false; // Reset to look for next section
}
}
} else {
delay(10); // Wait for buffer to fill
}
}
Serial.println("Stream Finished.");
} else {
Serial.print("HTTP Error: ");
Serial.println(httpCode);
}
http.end();
} else {
Serial.println("Connection Failed.");
}
}
}
// FETCHING_READINGS_UI