serial: Call set_custom_baudrate() thrice

Call the function before tcsetattr() settings are known, then again
with settings prepared but not yet applied and finally a third time
after tcsetattr().

Darwin support needs this change; there custom_baud code must be
called to modify the settings passed to tcsetattr() and then again
after tcsetattr() returns.

The change should be non-functional on all currently supported systems;
current code calls set_custom_baudrate() before any tcsetattr()
settings are prepared, so we have three stages in total.

This change originates from discussion of the macOS patch proposed by
Denis Ahrens in https://review.coreboot.org/c/flashrom/+/67822

Change-Id: I40cc443cfb7bf6b212b31826d437b898cc13c427
Signed-off-by: Peter Stuge <peter@stuge.se>
Reviewed-on: https://review.coreboot.org/c/flashrom/+/70569
Tested-by: build bot (Jenkins) <no-reply@coreboot.org>
Reviewed-by: Thomas Heijligen <src@posteo.de>

1 files changed, 14 insertions, 3 deletions

diff --git a/serial.c b/serial.c
index a0ef632f..b9ce2b38 100644
--- a/serial.c
+++ b/serial.c
@@ -179,6 +179,7 @@ int serialport_config(fdtype fd, int baud)
 	}
 	msg_pdbg("Baud rate is %ld.\n", dcb.BaudRate);
 #else
+	int custom_baud = (baud >= 0 && use_custom_baud(baud, sp_baudtable));
 	struct termios wanted, observed;
 	if (tcgetattr(fd, &observed) != 0) {
 		msg_perr_strerror("Could not fetch original serial port configuration: ");
@@ -186,8 +187,8 @@ int serialport_config(fdtype fd, int baud)
 	}
 	wanted = observed;
 	if (baud >= 0) {
-		if (use_custom_baud(baud, sp_baudtable)) {
-			if (set_custom_baudrate(fd, baud)) {
+		if (custom_baud) {
+			if (set_custom_baudrate(fd, baud, BEFORE_FLAGS, NULL)) {
 				msg_perr_strerror("Could not set custom baudrate: ");
 				return 1;
 			}
@@ -198,7 +199,6 @@ int serialport_config(fdtype fd, int baud)
 				msg_perr_strerror("Could not fetch serial port configuration: ");
 				return 1;
 			}
-			msg_pdbg("Using custom baud rate.\n");
 		} else {
 			const struct baudentry *entry = round_baud(baud);
 			if (cfsetispeed(&wanted, entry->flag) != 0 || cfsetospeed(&wanted, entry->flag) != 0) {
@@ -212,6 +212,10 @@ int serialport_config(fdtype fd, int baud)
 	wanted.c_lflag &= ~(ICANON | ECHO | ECHOE | ISIG | IEXTEN);
 	wanted.c_iflag &= ~(IXON | IXOFF | IXANY | ICRNL | IGNCR | INLCR);
 	wanted.c_oflag &= ~OPOST;
+	if (custom_baud && set_custom_baudrate(fd, baud, WITH_FLAGS, &wanted)) {
+		msg_perr_strerror("Could not set custom baudrate: ");
+		return 1;
+	}
 	if (tcsetattr(fd, TCSANOW, &wanted) != 0) {
 		msg_perr_strerror("Could not change serial port configuration: ");
 		return 1;
@@ -236,6 +240,13 @@ int serialport_config(fdtype fd, int baud)
 			 (long)observed.c_oflag, (long)wanted.c_oflag
 			);
 	}
+	if (custom_baud) {
+		if (set_custom_baudrate(fd, baud, AFTER_FLAGS, &wanted)) {
+			msg_perr_strerror("Could not set custom baudrate: ");
+			return 1;
+		}
+		msg_pdbg("Using custom baud rate.\n");
+	}
 	if (cfgetispeed(&observed) != cfgetispeed(&wanted) ||
 	    cfgetospeed(&observed) != cfgetospeed(&wanted)) {
 		msg_pwarn("Could not set baud rates exactly.\n");